2024-12-06 06:13:35,735 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-06 06:13:35,751 main DEBUG Took 0.013494 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 06:13:35,752 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 06:13:35,752 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 06:13:35,754 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 06:13:35,755 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,765 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 06:13:35,779 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,782 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,783 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,784 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,786 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,787 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,788 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,789 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,790 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,790 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,791 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,792 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,792 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,793 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,793 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,794 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 06:13:35,794 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,795 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 06:13:35,796 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 06:13:35,798 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 06:13:35,801 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 06:13:35,801 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 06:13:35,803 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 06:13:35,803 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 06:13:35,815 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 06:13:35,818 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 06:13:35,820 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 06:13:35,821 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 06:13:35,821 main DEBUG createAppenders(={Console}) 2024-12-06 06:13:35,822 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-06 06:13:35,823 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-06 06:13:35,823 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-06 06:13:35,824 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 06:13:35,824 main DEBUG OutputStream closed 2024-12-06 06:13:35,825 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 06:13:35,825 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 06:13:35,825 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-06 06:13:35,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 06:13:35,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 06:13:35,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 06:13:35,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 06:13:35,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 06:13:35,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 06:13:35,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 06:13:35,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 06:13:35,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 06:13:35,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 06:13:35,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 06:13:35,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 06:13:35,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 06:13:35,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 06:13:35,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 06:13:35,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 06:13:35,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 06:13:35,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 06:13:35,904 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 06:13:35,905 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-06 06:13:35,905 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 06:13:35,906 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-06T06:13:36,146 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c 2024-12-06 06:13:36,149 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 06:13:36,150 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T06:13:36,159 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-06T06:13:36,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-06T06:13:36,202 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/cluster_f2665540-2ae4-21bf-204f-a0e90cc9380d, deleteOnExit=true 2024-12-06T06:13:36,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-06T06:13:36,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/test.cache.data in system properties and HBase conf 2024-12-06T06:13:36,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T06:13:36,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/hadoop.log.dir in system properties and HBase conf 2024-12-06T06:13:36,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T06:13:36,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T06:13:36,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T06:13:36,303 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T06:13:36,395 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T06:13:36,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T06:13:36,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T06:13:36,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T06:13:36,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T06:13:36,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T06:13:36,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T06:13:36,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T06:13:36,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T06:13:36,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T06:13:36,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/nfs.dump.dir in system properties and HBase conf 2024-12-06T06:13:36,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/java.io.tmpdir in system properties and HBase conf 2024-12-06T06:13:36,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T06:13:36,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T06:13:36,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T06:13:37,224 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-06T06:13:37,306 INFO [Time-limited test {}] log.Log(170): Logging initialized @2257ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-06T06:13:37,383 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T06:13:37,447 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T06:13:37,467 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T06:13:37,467 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T06:13:37,468 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T06:13:37,480 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T06:13:37,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/hadoop.log.dir/,AVAILABLE} 2024-12-06T06:13:37,483 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T06:13:37,670 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/java.io.tmpdir/jetty-localhost-42571-hadoop-hdfs-3_4_1-tests_jar-_-any-10148910174292166757/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T06:13:37,683 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42571} 2024-12-06T06:13:37,683 INFO [Time-limited test {}] server.Server(415): Started @2634ms 2024-12-06T06:13:38,511 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T06:13:38,518 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T06:13:38,519 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T06:13:38,519 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T06:13:38,520 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T06:13:38,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/hadoop.log.dir/,AVAILABLE} 2024-12-06T06:13:38,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T06:13:38,641 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/java.io.tmpdir/jetty-localhost-44021-hadoop-hdfs-3_4_1-tests_jar-_-any-7953249371082392625/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T06:13:38,642 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:44021} 2024-12-06T06:13:38,642 INFO [Time-limited test {}] server.Server(415): Started @3593ms 2024-12-06T06:13:38,702 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T06:13:39,203 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/cluster_f2665540-2ae4-21bf-204f-a0e90cc9380d/data/data2/current/BP-840892825-172.17.0.2-1733465616971/current, will proceed with Du for space computation calculation, 2024-12-06T06:13:39,203 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/cluster_f2665540-2ae4-21bf-204f-a0e90cc9380d/data/data1/current/BP-840892825-172.17.0.2-1733465616971/current, will proceed with Du for space computation calculation, 2024-12-06T06:13:39,242 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T06:13:39,294 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c 2024-12-06T06:13:39,297 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x42f06060bd5c4ddd with lease ID 0xdcdaee0d9519649c: Processing first storage report for DS-a165a02b-557a-4a82-9655-31a6fd4e9b59 from datanode DatanodeRegistration(127.0.0.1:38749, datanodeUuid=b200ec0b-a77d-4fc5-b349-d290ec2f1a55, infoPort=40249, infoSecurePort=0, ipcPort=36743, storageInfo=lv=-57;cid=testClusterID;nsid=1264231266;c=1733465616971) 2024-12-06T06:13:39,298 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x42f06060bd5c4ddd with lease ID 0xdcdaee0d9519649c: from storage DS-a165a02b-557a-4a82-9655-31a6fd4e9b59 node DatanodeRegistration(127.0.0.1:38749, datanodeUuid=b200ec0b-a77d-4fc5-b349-d290ec2f1a55, infoPort=40249, infoSecurePort=0, ipcPort=36743, storageInfo=lv=-57;cid=testClusterID;nsid=1264231266;c=1733465616971), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T06:13:39,299 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x42f06060bd5c4ddd with lease ID 0xdcdaee0d9519649c: Processing first storage report for DS-ff0948b2-83d0-425d-9b8a-2820cb261014 from datanode DatanodeRegistration(127.0.0.1:38749, datanodeUuid=b200ec0b-a77d-4fc5-b349-d290ec2f1a55, infoPort=40249, infoSecurePort=0, ipcPort=36743, storageInfo=lv=-57;cid=testClusterID;nsid=1264231266;c=1733465616971) 2024-12-06T06:13:39,299 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x42f06060bd5c4ddd with lease ID 0xdcdaee0d9519649c: from storage DS-ff0948b2-83d0-425d-9b8a-2820cb261014 node DatanodeRegistration(127.0.0.1:38749, datanodeUuid=b200ec0b-a77d-4fc5-b349-d290ec2f1a55, infoPort=40249, infoSecurePort=0, ipcPort=36743, storageInfo=lv=-57;cid=testClusterID;nsid=1264231266;c=1733465616971), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T06:13:39,380 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/cluster_f2665540-2ae4-21bf-204f-a0e90cc9380d/zookeeper_0, clientPort=55017, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/cluster_f2665540-2ae4-21bf-204f-a0e90cc9380d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/cluster_f2665540-2ae4-21bf-204f-a0e90cc9380d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T06:13:39,390 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55017 2024-12-06T06:13:39,404 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T06:13:39,408 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T06:13:39,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741825_1001 (size=7) 2024-12-06T06:13:40,105 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea with version=8 2024-12-06T06:13:40,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/hbase-staging 2024-12-06T06:13:40,205 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T06:13:40,456 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6336e8474e1c:0 server-side Connection retries=45 2024-12-06T06:13:40,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T06:13:40,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T06:13:40,475 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T06:13:40,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T06:13:40,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T06:13:40,652 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T06:13:40,717 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T06:13:40,727 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T06:13:40,731 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T06:13:40,758 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3079 (auto-detected) 2024-12-06T06:13:40,759 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T06:13:40,778 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40959 2024-12-06T06:13:40,801 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40959 connecting to ZooKeeper ensemble=127.0.0.1:55017 2024-12-06T06:13:40,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:409590x0, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T06:13:40,839 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40959-0x1018668262d0000 connected 2024-12-06T06:13:40,868 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T06:13:40,870 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T06:13:40,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T06:13:40,884 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea, hbase.cluster.distributed=false 2024-12-06T06:13:40,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T06:13:40,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40959 2024-12-06T06:13:40,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40959 2024-12-06T06:13:40,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40959 2024-12-06T06:13:40,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40959 2024-12-06T06:13:40,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40959 2024-12-06T06:13:41,041 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6336e8474e1c:0 server-side Connection retries=45 2024-12-06T06:13:41,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T06:13:41,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T06:13:41,043 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T06:13:41,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T06:13:41,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T06:13:41,046 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T06:13:41,050 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T06:13:41,051 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39987 2024-12-06T06:13:41,054 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39987 connecting to ZooKeeper ensemble=127.0.0.1:55017 2024-12-06T06:13:41,056 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T06:13:41,060 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T06:13:41,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399870x0, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T06:13:41,078 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39987-0x1018668262d0001 connected 2024-12-06T06:13:41,079 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T06:13:41,085 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T06:13:41,095 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T06:13:41,098 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T06:13:41,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T06:13:41,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39987 2024-12-06T06:13:41,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39987 2024-12-06T06:13:41,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39987 2024-12-06T06:13:41,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39987 2024-12-06T06:13:41,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39987 2024-12-06T06:13:41,132 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6336e8474e1c:40959 2024-12-06T06:13:41,133 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6336e8474e1c,40959,1733465620259 2024-12-06T06:13:41,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T06:13:41,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T06:13:41,143 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6336e8474e1c,40959,1733465620259 2024-12-06T06:13:41,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:41,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T06:13:41,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:41,169 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T06:13:41,171 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6336e8474e1c,40959,1733465620259 from backup master directory 2024-12-06T06:13:41,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T06:13:41,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6336e8474e1c,40959,1733465620259 2024-12-06T06:13:41,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T06:13:41,175 WARN [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T06:13:41,175 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6336e8474e1c,40959,1733465620259 2024-12-06T06:13:41,177 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T06:13:41,179 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T06:13:41,261 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/hbase.id] with ID: 8824d7b1-a9b8-45af-a59f-d90ecce0f1c1 2024-12-06T06:13:41,262 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/.tmp/hbase.id 2024-12-06T06:13:41,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741826_1002 (size=42) 2024-12-06T06:13:41,677 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/.tmp/hbase.id]:[hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/hbase.id] 2024-12-06T06:13:41,719 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T06:13:41,724 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T06:13:41,746 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-06T06:13:41,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:41,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:41,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741827_1003 (size=196) 2024-12-06T06:13:41,789 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:13:41,794 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T06:13:41,818 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:41,825 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T06:13:41,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741828_1004 (size=1189) 2024-12-06T06:13:42,287 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store 2024-12-06T06:13:42,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741829_1005 (size=34) 2024-12-06T06:13:42,714 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T06:13:42,718 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:13:42,719 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T06:13:42,719 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T06:13:42,720 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T06:13:42,721 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T06:13:42,722 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T06:13:42,722 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T06:13:42,723 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733465622719Disabling compacts and flushes for region at 1733465622719Disabling writes for close at 1733465622721 (+2 ms)Writing region close event to WAL at 1733465622722 (+1 ms)Closed at 1733465622722 2024-12-06T06:13:42,725 WARN [master/6336e8474e1c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/.initializing 2024-12-06T06:13:42,725 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/WALs/6336e8474e1c,40959,1733465620259 2024-12-06T06:13:42,735 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T06:13:42,750 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6336e8474e1c%2C40959%2C1733465620259, suffix=, logDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/WALs/6336e8474e1c,40959,1733465620259, archiveDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/oldWALs, maxLogs=10 2024-12-06T06:13:42,773 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/WALs/6336e8474e1c,40959,1733465620259/6336e8474e1c%2C40959%2C1733465620259.1733465622755, exclude list is [], retry=0 2024-12-06T06:13:42,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38749,DS-a165a02b-557a-4a82-9655-31a6fd4e9b59,DISK] 2024-12-06T06:13:42,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-06T06:13:42,830 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/WALs/6336e8474e1c,40959,1733465620259/6336e8474e1c%2C40959%2C1733465620259.1733465622755 2024-12-06T06:13:42,831 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-06T06:13:42,832 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:13:42,832 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:13:42,836 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,837 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,908 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T06:13:42,912 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:42,915 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:42,915 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T06:13:42,919 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:42,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:13:42,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T06:13:42,924 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:42,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:13:42,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T06:13:42,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:42,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:13:42,929 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,933 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,935 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,940 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,941 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,944 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T06:13:42,948 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T06:13:42,953 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T06:13:42,954 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58957493, jitterRate=-0.12146489322185516}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T06:13:42,959 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733465622850Initializing all the Stores at 1733465622852 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733465622853 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465622853Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465622854 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465622854Cleaning up temporary data from old regions at 1733465622941 (+87 ms)Region opened successfully at 1733465622959 (+18 ms) 2024-12-06T06:13:42,960 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T06:13:42,993 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f684b27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6336e8474e1c/172.17.0.2:0 2024-12-06T06:13:43,025 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T06:13:43,036 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T06:13:43,037 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T06:13:43,040 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T06:13:43,041 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T06:13:43,046 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-06T06:13:43,046 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T06:13:43,073 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T06:13:43,081 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T06:13:43,083 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T06:13:43,085 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T06:13:43,087 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T06:13:43,089 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T06:13:43,091 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T06:13:43,094 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T06:13:43,096 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T06:13:43,097 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T06:13:43,099 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T06:13:43,117 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T06:13:43,118 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T06:13:43,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T06:13:43,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T06:13:43,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:43,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:43,128 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6336e8474e1c,40959,1733465620259, sessionid=0x1018668262d0000, setting cluster-up flag (Was=false) 2024-12-06T06:13:43,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:43,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:43,156 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T06:13:43,158 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6336e8474e1c,40959,1733465620259 2024-12-06T06:13:43,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:43,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:43,172 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T06:13:43,174 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6336e8474e1c,40959,1733465620259 2024-12-06T06:13:43,181 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T06:13:43,219 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(746): ClusterId : 8824d7b1-a9b8-45af-a59f-d90ecce0f1c1 2024-12-06T06:13:43,222 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T06:13:43,228 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T06:13:43,228 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T06:13:43,233 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T06:13:43,233 DEBUG [RS:0;6336e8474e1c:39987 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2791c47e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6336e8474e1c/172.17.0.2:0 2024-12-06T06:13:43,250 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6336e8474e1c:39987 2024-12-06T06:13:43,255 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T06:13:43,255 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T06:13:43,255 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T06:13:43,258 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(2659): reportForDuty to master=6336e8474e1c,40959,1733465620259 with port=39987, startcode=1733465620995 2024-12-06T06:13:43,266 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T06:13:43,271 DEBUG [RS:0;6336e8474e1c:39987 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T06:13:43,277 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T06:13:43,284 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T06:13:43,292 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6336e8474e1c,40959,1733465620259 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T06:13:43,301 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6336e8474e1c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T06:13:43,302 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6336e8474e1c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T06:13:43,302 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6336e8474e1c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T06:13:43,302 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6336e8474e1c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T06:13:43,302 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6336e8474e1c:0, corePoolSize=10, maxPoolSize=10 2024-12-06T06:13:43,303 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,303 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6336e8474e1c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T06:13:43,303 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,306 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733465653306 2024-12-06T06:13:43,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49553, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T06:13:43,308 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T06:13:43,309 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T06:13:43,310 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T06:13:43,311 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T06:13:43,314 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T06:13:43,314 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T06:13:43,315 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T06:13:43,315 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T06:13:43,315 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40959 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:43,317 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:43,317 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T06:13:43,316 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,321 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T06:13:43,322 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T06:13:43,323 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T06:13:43,337 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T06:13:43,337 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T06:13:43,339 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6336e8474e1c:0:becomeActiveMaster-HFileCleaner.large.0-1733465623339,5,FailOnTimeoutGroup] 2024-12-06T06:13:43,340 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6336e8474e1c:0:becomeActiveMaster-HFileCleaner.small.0-1733465623340,5,FailOnTimeoutGroup] 2024-12-06T06:13:43,340 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,341 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T06:13:43,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741831_1007 (size=1321) 2024-12-06T06:13:43,342 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,342 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,355 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T06:13:43,355 WARN [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T06:13:43,457 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(2659): reportForDuty to master=6336e8474e1c,40959,1733465620259 with port=39987, startcode=1733465620995 2024-12-06T06:13:43,460 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40959 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6336e8474e1c,39987,1733465620995 2024-12-06T06:13:43,463 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40959 {}] master.ServerManager(517): Registering regionserver=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:43,472 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea 2024-12-06T06:13:43,472 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37641 2024-12-06T06:13:43,472 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T06:13:43,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T06:13:43,477 DEBUG [RS:0;6336e8474e1c:39987 {}] zookeeper.ZKUtil(111): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6336e8474e1c,39987,1733465620995 2024-12-06T06:13:43,477 WARN [RS:0;6336e8474e1c:39987 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T06:13:43,478 INFO [RS:0;6336e8474e1c:39987 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T06:13:43,478 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/WALs/6336e8474e1c,39987,1733465620995 2024-12-06T06:13:43,479 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6336e8474e1c,39987,1733465620995] 2024-12-06T06:13:43,504 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T06:13:43,519 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T06:13:43,524 INFO [RS:0;6336e8474e1c:39987 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T06:13:43,524 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,525 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T06:13:43,531 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T06:13:43,532 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,532 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,533 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,533 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,533 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,533 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,533 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6336e8474e1c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T06:13:43,533 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,533 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,534 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,534 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,534 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,534 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6336e8474e1c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T06:13:43,534 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6336e8474e1c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T06:13:43,534 DEBUG [RS:0;6336e8474e1c:39987 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T06:13:43,535 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,535 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,536 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,536 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,536 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,536 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=6336e8474e1c,39987,1733465620995-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T06:13:43,557 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T06:13:43,559 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=6336e8474e1c,39987,1733465620995-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,560 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,560 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.Replication(171): 6336e8474e1c,39987,1733465620995 started 2024-12-06T06:13:43,579 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:43,580 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1482): Serving as 6336e8474e1c,39987,1733465620995, RpcServer on 6336e8474e1c/172.17.0.2:39987, sessionid=0x1018668262d0001 2024-12-06T06:13:43,581 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T06:13:43,581 DEBUG [RS:0;6336e8474e1c:39987 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6336e8474e1c,39987,1733465620995 2024-12-06T06:13:43,582 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6336e8474e1c,39987,1733465620995' 2024-12-06T06:13:43,582 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T06:13:43,583 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T06:13:43,584 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T06:13:43,584 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T06:13:43,584 DEBUG [RS:0;6336e8474e1c:39987 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6336e8474e1c,39987,1733465620995 2024-12-06T06:13:43,584 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6336e8474e1c,39987,1733465620995' 2024-12-06T06:13:43,585 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T06:13:43,588 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T06:13:43,589 DEBUG [RS:0;6336e8474e1c:39987 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T06:13:43,589 INFO [RS:0;6336e8474e1c:39987 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T06:13:43,589 INFO [RS:0;6336e8474e1c:39987 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T06:13:43,697 INFO [RS:0;6336e8474e1c:39987 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T06:13:43,701 INFO [RS:0;6336e8474e1c:39987 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6336e8474e1c%2C39987%2C1733465620995, suffix=, logDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/WALs/6336e8474e1c,39987,1733465620995, archiveDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/oldWALs, maxLogs=32 2024-12-06T06:13:43,721 DEBUG [RS:0;6336e8474e1c:39987 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/WALs/6336e8474e1c,39987,1733465620995/6336e8474e1c%2C39987%2C1733465620995.1733465623703, exclude list is [], retry=0 2024-12-06T06:13:43,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38749,DS-a165a02b-557a-4a82-9655-31a6fd4e9b59,DISK] 2024-12-06T06:13:43,734 INFO [RS:0;6336e8474e1c:39987 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/WALs/6336e8474e1c,39987,1733465620995/6336e8474e1c%2C39987%2C1733465620995.1733465623703 2024-12-06T06:13:43,734 DEBUG [RS:0;6336e8474e1c:39987 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-06T06:13:43,744 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T06:13:43,745 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea 2024-12-06T06:13:43,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741833_1009 (size=32) 2024-12-06T06:13:44,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:13:44,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T06:13:44,184 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T06:13:44,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:44,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:44,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T06:13:44,189 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T06:13:44,189 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:44,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:44,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T06:13:44,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T06:13:44,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:44,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:44,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T06:13:44,202 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T06:13:44,202 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:44,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:44,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T06:13:44,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740 2024-12-06T06:13:44,208 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740 2024-12-06T06:13:44,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T06:13:44,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T06:13:44,213 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:13:44,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T06:13:44,221 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T06:13:44,223 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71874371, jitterRate=0.07101158797740936}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:13:44,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733465624168Initializing all the Stores at 1733465624171 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733465624171Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733465624175 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465624175Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733465624175Cleaning up temporary data from old regions at 1733465624212 (+37 ms)Region opened successfully at 1733465624226 (+14 ms) 2024-12-06T06:13:44,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T06:13:44,227 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T06:13:44,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T06:13:44,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T06:13:44,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T06:13:44,229 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T06:13:44,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733465624226Disabling compacts and flushes for region at 1733465624226Disabling writes for close at 1733465624227 (+1 ms)Writing region close event to WAL at 1733465624229 (+2 ms)Closed at 1733465624229 2024-12-06T06:13:44,234 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T06:13:44,235 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T06:13:44,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T06:13:44,256 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T06:13:44,260 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T06:13:44,413 DEBUG [6336e8474e1c:40959 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-06T06:13:44,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:44,436 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6336e8474e1c,39987,1733465620995, state=OPENING 2024-12-06T06:13:44,442 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T06:13:44,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:44,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:13:44,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T06:13:44,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T06:13:44,449 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T06:13:44,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:13:44,639 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T06:13:44,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T06:13:44,662 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T06:13:44,663 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T06:13:44,663 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T06:13:44,667 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6336e8474e1c%2C39987%2C1733465620995.meta, suffix=.meta, logDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/WALs/6336e8474e1c,39987,1733465620995, archiveDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/oldWALs, maxLogs=32 2024-12-06T06:13:44,684 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/WALs/6336e8474e1c,39987,1733465620995/6336e8474e1c%2C39987%2C1733465620995.meta.1733465624669.meta, exclude list is [], retry=0 2024-12-06T06:13:44,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38749,DS-a165a02b-557a-4a82-9655-31a6fd4e9b59,DISK] 2024-12-06T06:13:44,693 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/WALs/6336e8474e1c,39987,1733465620995/6336e8474e1c%2C39987%2C1733465620995.meta.1733465624669.meta 2024-12-06T06:13:44,693 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-06T06:13:44,694 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:13:44,696 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T06:13:44,699 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T06:13:44,704 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T06:13:44,709 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T06:13:44,709 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:13:44,710 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T06:13:44,710 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T06:13:44,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T06:13:44,714 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T06:13:44,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:44,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:44,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T06:13:44,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T06:13:44,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:44,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:44,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T06:13:44,720 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T06:13:44,720 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:44,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:44,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T06:13:44,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T06:13:44,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:44,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T06:13:44,723 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T06:13:44,725 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740 2024-12-06T06:13:44,728 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740 2024-12-06T06:13:44,730 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T06:13:44,730 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T06:13:44,731 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:13:44,734 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T06:13:44,736 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62939462, jitterRate=-0.062128931283950806}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:13:44,736 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T06:13:44,738 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733465624710Writing region info on filesystem at 1733465624711 (+1 ms)Initializing all the Stores at 1733465624712 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733465624713 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733465624713Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465624713Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733465624713Cleaning up temporary data from old regions at 1733465624730 (+17 ms)Running coprocessor post-open hooks at 1733465624736 (+6 ms)Region opened successfully at 1733465624738 (+2 ms) 2024-12-06T06:13:44,745 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733465624627 2024-12-06T06:13:44,758 DEBUG [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T06:13:44,758 INFO [RS_OPEN_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T06:13:44,760 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:44,762 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6336e8474e1c,39987,1733465620995, state=OPEN 2024-12-06T06:13:44,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T06:13:44,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T06:13:44,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T06:13:44,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T06:13:44,771 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:44,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T06:13:44,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6336e8474e1c,39987,1733465620995 in 320 msec 2024-12-06T06:13:44,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T06:13:44,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 535 msec 2024-12-06T06:13:44,784 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T06:13:44,784 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T06:13:44,808 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:44,809 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:44,832 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:44,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35965, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:44,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6420 sec 2024-12-06T06:13:44,855 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733465624854, completionTime=-1 2024-12-06T06:13:44,857 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-06T06:13:44,857 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T06:13:44,884 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-06T06:13:44,885 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733465684884 2024-12-06T06:13:44,885 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733465744885 2024-12-06T06:13:44,885 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-06T06:13:44,887 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6336e8474e1c,40959,1733465620259-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:44,887 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6336e8474e1c,40959,1733465620259-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:44,887 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6336e8474e1c,40959,1733465620259-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:44,889 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6336e8474e1c:40959, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:44,889 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:44,890 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:44,895 DEBUG [master/6336e8474e1c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T06:13:44,922 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.747sec 2024-12-06T06:13:44,923 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T06:13:44,925 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T06:13:44,926 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T06:13:44,926 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T06:13:44,926 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T06:13:44,927 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6336e8474e1c,40959,1733465620259-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T06:13:44,928 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6336e8474e1c,40959,1733465620259-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T06:13:44,936 DEBUG [master/6336e8474e1c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T06:13:44,937 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T06:13:44,937 INFO [master/6336e8474e1c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6336e8474e1c,40959,1733465620259-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T06:13:45,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e025496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:45,039 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T06:13:45,039 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T06:13:45,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:45,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:45,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a0a815, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:45,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:45,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:45,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2c026e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:45,067 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:45,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:45,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:45,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6336e8474e1c,40959,1733465620259 2024-12-06T06:13:45,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=5332 2024-12-06T06:13:45,120 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T06:13:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6336e8474e1c,40959,1733465620259 2024-12-06T06:13:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@674ec8b 2024-12-06T06:13:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T06:13:45,154 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T06:13:45,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:13:45,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:13:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T06:13:45,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T06:13:45,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-06T06:13:45,175 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:45,178 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T06:13:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T06:13:45,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741835_1011 (size=1061) 2024-12-06T06:13:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T06:13:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T06:13:45,594 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea 2024-12-06T06:13:45,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741836_1012 (size=53) 2024-12-06T06:13:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T06:13:46,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:13:46,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3860e7c2497c4799e160cda168e0086d, disabling compactions & flushes 2024-12-06T06:13:46,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:46,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:46,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. after waiting 0 ms 2024-12-06T06:13:46,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:46,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:46,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3860e7c2497c4799e160cda168e0086d: Waiting for close lock at 1733465626004Disabling compacts and flushes for region at 1733465626004Disabling writes for close at 1733465626004Writing region close event to WAL at 1733465626004Closed at 1733465626004 2024-12-06T06:13:46,007 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T06:13:46,013 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733465626007"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733465626007"}]},"ts":"1733465626007"} 2024-12-06T06:13:46,017 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T06:13:46,020 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T06:13:46,023 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465626020"}]},"ts":"1733465626020"} 2024-12-06T06:13:46,028 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T06:13:46,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, ASSIGN}] 2024-12-06T06:13:46,032 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, ASSIGN 2024-12-06T06:13:46,035 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, ASSIGN; state=OFFLINE, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=false 2024-12-06T06:13:46,187 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3860e7c2497c4799e160cda168e0086d, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:46,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, ASSIGN because future has completed 2024-12-06T06:13:46,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:13:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T06:13:46,353 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:46,353 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:13:46,354 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,354 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:13:46,354 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,354 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,356 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,360 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:13:46,360 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3860e7c2497c4799e160cda168e0086d columnFamilyName A 2024-12-06T06:13:46,360 DEBUG [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:46,361 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.HStore(327): Store=3860e7c2497c4799e160cda168e0086d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:13:46,361 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,363 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:13:46,363 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3860e7c2497c4799e160cda168e0086d columnFamilyName B 2024-12-06T06:13:46,364 DEBUG [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:46,364 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.HStore(327): Store=3860e7c2497c4799e160cda168e0086d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:13:46,364 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,366 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:13:46,366 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3860e7c2497c4799e160cda168e0086d columnFamilyName C 2024-12-06T06:13:46,367 DEBUG [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:13:46,367 INFO [StoreOpener-3860e7c2497c4799e160cda168e0086d-1 {}] regionserver.HStore(327): Store=3860e7c2497c4799e160cda168e0086d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:13:46,368 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:46,368 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,369 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,370 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,371 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,372 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,372 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:13:46,375 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,378 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T06:13:46,379 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3860e7c2497c4799e160cda168e0086d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63917275, jitterRate=-0.04755838215351105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:13:46,379 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:46,380 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3860e7c2497c4799e160cda168e0086d: Running coprocessor pre-open hook at 1733465626354Writing region info on filesystem at 1733465626354Initializing all the Stores at 1733465626356 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465626356Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465626356Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465626356Cleaning up temporary data from old regions at 1733465626372 (+16 ms)Running coprocessor post-open hooks at 1733465626379 (+7 ms)Region opened successfully at 1733465626379 2024-12-06T06:13:46,382 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., pid=6, masterSystemTime=1733465626347 2024-12-06T06:13:46,386 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:46,386 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:46,386 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3860e7c2497c4799e160cda168e0086d, regionState=OPEN, openSeqNum=2, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:46,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:13:46,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T06:13:46,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 in 201 msec 2024-12-06T06:13:46,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T06:13:46,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, ASSIGN in 367 msec 2024-12-06T06:13:46,403 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T06:13:46,403 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465626403"}]},"ts":"1733465626403"} 2024-12-06T06:13:46,407 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T06:13:46,408 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T06:13:46,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2450 sec 2024-12-06T06:13:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T06:13:47,345 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:13:47,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c267de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,355 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1a1116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,358 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da04ac8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,360 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,366 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T06:13:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6336e8474e1c,40959,1733465620259 2024-12-06T06:13:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5840dd2c 2024-12-06T06:13:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T06:13:47,371 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T06:13:47,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,376 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be8c6a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,379 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6a98a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,383 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7601b089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,389 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a1762b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,394 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93ecac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,397 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6611efce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ec8db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,401 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b25c90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0534ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,408 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1377b918, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,410 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e73fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6fee22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,417 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4094989e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,420 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a56f43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,422 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612fe67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdd76a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,426 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3cf0fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b1e9af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e2441c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:13:47,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:13:47,434 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:13:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:13:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:13:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2269a7fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:13:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:13:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:13:47,437 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:13:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bde23a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:13:47,442 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,442 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,442 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,443 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,443 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,447 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:13:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,450 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-06T06:13:47,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T06:13:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,460 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,467 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:13:47,470 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:13:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:13:47,479 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:13:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:13:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:13:47,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:13:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:13:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:47,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:13:47,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T06:13:47,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:47,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:47,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:47,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:47,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:47,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/0b456b12b0b6409da957aeaf3ee0a5e0 is 50, key is test_row_1/A:col10/1733465627520/Put/seqid=0 2024-12-06T06:13:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465687630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465687645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465687654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465687655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733465627581, max timestamp=1733465627653Num uniques 859; 2024-12-06T06:13:47,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733465627581, max timestamp=1733465627653Num uniques 859; 2024-12-06T06:13:47,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T06:13:47,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T06:13:47,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733465627581, max timestamp=1733465627653Num uniques 859; 2024-12-06T06:13:47,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T06:13:47,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741837_1013 (size=11997) 2024-12-06T06:13:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T06:13:47,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465687813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:47,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465687818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465687819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465687825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465687825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:47,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:47,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465687935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465687946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465687945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465687946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465687954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T06:13:48,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/0b456b12b0b6409da957aeaf3ee0a5e0 2024-12-06T06:13:48,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465688148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:48,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:48,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465688165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465688165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465688166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465688175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5eae4b1f46ae4928bd5a820861ff7384 is 50, key is test_row_1/B:col10/1733465627520/Put/seqid=0 2024-12-06T06:13:48,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:48,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:48,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741838_1014 (size=9657) 2024-12-06T06:13:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465688465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:48,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:48,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465688484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465688485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465688494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465688495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T06:13:48,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:48,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:48,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5eae4b1f46ae4928bd5a820861ff7384 2024-12-06T06:13:48,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/ae4b8c75123d4d909ea0fcf7638cf3ef is 50, key is test_row_1/C:col10/1733465627520/Put/seqid=0 2024-12-06T06:13:48,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:48,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741839_1015 (size=9657) 2024-12-06T06:13:48,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/ae4b8c75123d4d909ea0fcf7638cf3ef 2024-12-06T06:13:48,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/0b456b12b0b6409da957aeaf3ee0a5e0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/0b456b12b0b6409da957aeaf3ee0a5e0 2024-12-06T06:13:48,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/0b456b12b0b6409da957aeaf3ee0a5e0, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T06:13:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5eae4b1f46ae4928bd5a820861ff7384 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5eae4b1f46ae4928bd5a820861ff7384 2024-12-06T06:13:48,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5eae4b1f46ae4928bd5a820861ff7384, entries=100, sequenceid=12, filesize=9.4 K 2024-12-06T06:13:48,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/ae4b8c75123d4d909ea0fcf7638cf3ef as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ae4b8c75123d4d909ea0fcf7638cf3ef 2024-12-06T06:13:48,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ae4b8c75123d4d909ea0fcf7638cf3ef, entries=100, sequenceid=12, filesize=9.4 K 2024-12-06T06:13:48,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 3860e7c2497c4799e160cda168e0086d in 1391ms, sequenceid=12, compaction requested=false 2024-12-06T06:13:48,917 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T06:13:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:48,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T06:13:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:48,951 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-12-06T06:13:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:48,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:49,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/cd81269cb2404307a1331c877dd0dade is 50, key is test_row_0/A:col10/1733465627819/Put/seqid=0 2024-12-06T06:13:49,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741840_1016 (size=12001) 2024-12-06T06:13:49,048 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/cd81269cb2404307a1331c877dd0dade 2024-12-06T06:13:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465689020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465689024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465689036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465689040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465689044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b37fc7c43783471f89a8745d6b6a4872 is 50, key is test_row_0/B:col10/1733465627819/Put/seqid=0 2024-12-06T06:13:49,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741841_1017 (size=12001) 2024-12-06T06:13:49,118 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b37fc7c43783471f89a8745d6b6a4872 2024-12-06T06:13:49,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/aa025e7e281b4b4f86e00b6cfd7fb2f1 is 50, key is test_row_0/C:col10/1733465627819/Put/seqid=0 2024-12-06T06:13:49,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465689165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465689165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465689165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741842_1018 (size=12001) 2024-12-06T06:13:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,182 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/aa025e7e281b4b4f86e00b6cfd7fb2f1 2024-12-06T06:13:49,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465689166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465689166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/cd81269cb2404307a1331c877dd0dade as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd81269cb2404307a1331c877dd0dade 2024-12-06T06:13:49,210 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd81269cb2404307a1331c877dd0dade, entries=150, sequenceid=45, filesize=11.7 K 2024-12-06T06:13:49,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b37fc7c43783471f89a8745d6b6a4872 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b37fc7c43783471f89a8745d6b6a4872 2024-12-06T06:13:49,250 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b37fc7c43783471f89a8745d6b6a4872, entries=150, sequenceid=45, filesize=11.7 K 2024-12-06T06:13:49,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/aa025e7e281b4b4f86e00b6cfd7fb2f1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/aa025e7e281b4b4f86e00b6cfd7fb2f1 2024-12-06T06:13:49,272 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/aa025e7e281b4b4f86e00b6cfd7fb2f1, entries=150, sequenceid=45, filesize=11.7 K 2024-12-06T06:13:49,276 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for 3860e7c2497c4799e160cda168e0086d in 324ms, sequenceid=45, compaction requested=false 2024-12-06T06:13:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-06T06:13:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-06T06:13:49,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T06:13:49,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8100 sec 2024-12-06T06:13:49,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8430 sec 2024-12-06T06:13:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:49,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T06:13:49,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:49,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:49,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:49,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:49,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:49,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:49,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/915c4d8803d940c88ecd64d2b74f0d57 is 50, key is test_row_0/A:col10/1733465629396/Put/seqid=0 2024-12-06T06:13:49,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741843_1019 (size=12001) 2024-12-06T06:13:49,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/915c4d8803d940c88ecd64d2b74f0d57 2024-12-06T06:13:49,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465689466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465689468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465689471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465689479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465689474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/03ab45337cdf4e48a8b25001f5cd5da2 is 50, key is test_row_0/B:col10/1733465629396/Put/seqid=0 2024-12-06T06:13:49,497 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T06:13:49,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741844_1020 (size=12001) 2024-12-06T06:13:49,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/03ab45337cdf4e48a8b25001f5cd5da2 2024-12-06T06:13:49,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/e58d25b7cbc24b8dac415aaf04ef6ecd is 50, key is test_row_0/C:col10/1733465629396/Put/seqid=0 2024-12-06T06:13:49,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741845_1021 (size=12001) 2024-12-06T06:13:49,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/e58d25b7cbc24b8dac415aaf04ef6ecd 2024-12-06T06:13:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465689595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465689596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465689596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465689595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465689596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T06:13:49,605 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:13:49,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/915c4d8803d940c88ecd64d2b74f0d57 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/915c4d8803d940c88ecd64d2b74f0d57 2024-12-06T06:13:49,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:13:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-06T06:13:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T06:13:49,612 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:13:49,616 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:13:49,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:13:49,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/915c4d8803d940c88ecd64d2b74f0d57, entries=150, sequenceid=58, filesize=11.7 K 2024-12-06T06:13:49,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/03ab45337cdf4e48a8b25001f5cd5da2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ab45337cdf4e48a8b25001f5cd5da2 2024-12-06T06:13:49,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ab45337cdf4e48a8b25001f5cd5da2, entries=150, sequenceid=58, filesize=11.7 K 2024-12-06T06:13:49,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/e58d25b7cbc24b8dac415aaf04ef6ecd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e58d25b7cbc24b8dac415aaf04ef6ecd 2024-12-06T06:13:49,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e58d25b7cbc24b8dac415aaf04ef6ecd, entries=150, sequenceid=58, filesize=11.7 K 2024-12-06T06:13:49,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3860e7c2497c4799e160cda168e0086d in 252ms, sequenceid=58, compaction requested=true 2024-12-06T06:13:49,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:49,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:49,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:49,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:49,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:49,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:49,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:49,661 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:49,663 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:49,666 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:49,667 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:13:49,668 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:49,669 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5eae4b1f46ae4928bd5a820861ff7384, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b37fc7c43783471f89a8745d6b6a4872, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ab45337cdf4e48a8b25001f5cd5da2] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=32.9 K 2024-12-06T06:13:49,669 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:49,669 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:49,669 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:49,670 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/0b456b12b0b6409da957aeaf3ee0a5e0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd81269cb2404307a1331c877dd0dade, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/915c4d8803d940c88ecd64d2b74f0d57] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=35.2 K 2024-12-06T06:13:49,670 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eae4b1f46ae4928bd5a820861ff7384, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733465627505 2024-12-06T06:13:49,670 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b456b12b0b6409da957aeaf3ee0a5e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733465627504 2024-12-06T06:13:49,671 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b37fc7c43783471f89a8745d6b6a4872, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733465627812 2024-12-06T06:13:49,671 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd81269cb2404307a1331c877dd0dade, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733465627812 2024-12-06T06:13:49,672 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ab45337cdf4e48a8b25001f5cd5da2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733465629016 2024-12-06T06:13:49,672 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915c4d8803d940c88ecd64d2b74f0d57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733465629016 2024-12-06T06:13:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T06:13:49,733 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#9 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:49,734 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5ac56220f8514825924d12f6fdee3f45 is 50, key is test_row_0/B:col10/1733465629396/Put/seqid=0 2024-12-06T06:13:49,745 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:49,747 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/74727690ca4641d8a26f91279b68b50a is 50, key is test_row_0/A:col10/1733465629396/Put/seqid=0 2024-12-06T06:13:49,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741846_1022 (size=12104) 2024-12-06T06:13:49,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T06:13:49,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:49,771 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T06:13:49,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:49,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:49,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:49,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:49,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741847_1023 (size=12104) 2024-12-06T06:13:49,792 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5ac56220f8514825924d12f6fdee3f45 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5ac56220f8514825924d12f6fdee3f45 2024-12-06T06:13:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/26932b711a5344c481cc333feb3d995c is 50, key is test_row_0/A:col10/1733465629471/Put/seqid=0 2024-12-06T06:13:49,803 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/74727690ca4641d8a26f91279b68b50a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/74727690ca4641d8a26f91279b68b50a 2024-12-06T06:13:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:49,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:49,826 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 74727690ca4641d8a26f91279b68b50a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:49,826 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:49,829 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=13, startTime=1733465629657; duration=0sec 2024-12-06T06:13:49,830 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:49,830 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:49,830 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:49,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741848_1024 (size=12001) 2024-12-06T06:13:49,833 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 5ac56220f8514825924d12f6fdee3f45(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:49,833 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:49,834 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=13, startTime=1733465629661; duration=0sec 2024-12-06T06:13:49,834 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:49,834 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:49,837 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:49,837 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:49,838 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/26932b711a5344c481cc333feb3d995c 2024-12-06T06:13:49,838 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:49,838 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ae4b8c75123d4d909ea0fcf7638cf3ef, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/aa025e7e281b4b4f86e00b6cfd7fb2f1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e58d25b7cbc24b8dac415aaf04ef6ecd] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=32.9 K 2024-12-06T06:13:49,840 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae4b8c75123d4d909ea0fcf7638cf3ef, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733465627505 2024-12-06T06:13:49,846 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa025e7e281b4b4f86e00b6cfd7fb2f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733465627812 2024-12-06T06:13:49,848 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e58d25b7cbc24b8dac415aaf04ef6ecd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733465629016 2024-12-06T06:13:49,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/3af5629cb3fc49eeab9b1cdd0a2b9615 is 50, key is test_row_0/B:col10/1733465629471/Put/seqid=0 2024-12-06T06:13:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465689854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465689859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465689860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465689864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465689864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741849_1025 (size=12001) 2024-12-06T06:13:49,889 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:49,890 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/483a7bc299374804bbc19b5b93e340e4 is 50, key is test_row_0/C:col10/1733465629396/Put/seqid=0 2024-12-06T06:13:49,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741850_1026 (size=12104) 2024-12-06T06:13:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T06:13:49,926 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/483a7bc299374804bbc19b5b93e340e4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/483a7bc299374804bbc19b5b93e340e4 2024-12-06T06:13:49,940 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into 483a7bc299374804bbc19b5b93e340e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:49,940 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:49,940 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=13, startTime=1733465629661; duration=0sec 2024-12-06T06:13:49,940 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:49,940 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465689974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465689986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465689986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465689986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465689986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465690184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465690194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465690193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465690194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465690194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,222 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T06:13:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T06:13:50,286 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/3af5629cb3fc49eeab9b1cdd0a2b9615 2024-12-06T06:13:50,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/8033da6b873945df93089eba6645bba0 is 50, key is test_row_0/C:col10/1733465629471/Put/seqid=0 2024-12-06T06:13:50,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741851_1027 (size=12001) 2024-12-06T06:13:50,339 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/8033da6b873945df93089eba6645bba0 2024-12-06T06:13:50,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/26932b711a5344c481cc333feb3d995c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/26932b711a5344c481cc333feb3d995c 2024-12-06T06:13:50,375 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/26932b711a5344c481cc333feb3d995c, entries=150, sequenceid=77, filesize=11.7 K 2024-12-06T06:13:50,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/3af5629cb3fc49eeab9b1cdd0a2b9615 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3af5629cb3fc49eeab9b1cdd0a2b9615 2024-12-06T06:13:50,394 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3af5629cb3fc49eeab9b1cdd0a2b9615, entries=150, sequenceid=77, filesize=11.7 K 2024-12-06T06:13:50,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/8033da6b873945df93089eba6645bba0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8033da6b873945df93089eba6645bba0 2024-12-06T06:13:50,411 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8033da6b873945df93089eba6645bba0, entries=150, sequenceid=77, filesize=11.7 K 2024-12-06T06:13:50,415 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 3860e7c2497c4799e160cda168e0086d in 643ms, sequenceid=77, compaction requested=false 2024-12-06T06:13:50,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:50,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:50,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-06T06:13:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-06T06:13:50,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-06T06:13:50,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 803 msec 2024-12-06T06:13:50,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 817 msec 2024-12-06T06:13:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:50,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:13:50,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:50,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:50,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:50,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:50,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:50,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:50,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4781812c93c5412088f14929e57bdb48 is 50, key is test_row_0/A:col10/1733465630505/Put/seqid=0 2024-12-06T06:13:50,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741852_1028 (size=14341) 2024-12-06T06:13:50,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4781812c93c5412088f14929e57bdb48 2024-12-06T06:13:50,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465690541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465690546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465690550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5875754dafd446cb8aa83c2038d469c3 is 50, key is test_row_0/B:col10/1733465630505/Put/seqid=0 2024-12-06T06:13:50,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465690554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465690558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741853_1029 (size=12001) 2024-12-06T06:13:50,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5875754dafd446cb8aa83c2038d469c3 2024-12-06T06:13:50,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1e7e2726d84f4e0d9888a14e3e2c25d5 is 50, key is test_row_0/C:col10/1733465630505/Put/seqid=0 2024-12-06T06:13:50,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465690664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465690665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465690665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465690676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465690678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741854_1030 (size=12001) 2024-12-06T06:13:50,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1e7e2726d84f4e0d9888a14e3e2c25d5 2024-12-06T06:13:50,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4781812c93c5412088f14929e57bdb48 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4781812c93c5412088f14929e57bdb48 2024-12-06T06:13:50,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T06:13:50,714 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-06T06:13:50,717 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T06:13:50,717 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-06T06:13:50,717 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-06T06:13:50,717 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-06T06:13:50,719 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T06:13:50,719 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-06T06:13:50,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4781812c93c5412088f14929e57bdb48, entries=200, sequenceid=95, filesize=14.0 K 2024-12-06T06:13:50,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5875754dafd446cb8aa83c2038d469c3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5875754dafd446cb8aa83c2038d469c3 2024-12-06T06:13:50,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5875754dafd446cb8aa83c2038d469c3, entries=150, sequenceid=95, filesize=11.7 K 2024-12-06T06:13:50,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1e7e2726d84f4e0d9888a14e3e2c25d5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e7e2726d84f4e0d9888a14e3e2c25d5 2024-12-06T06:13:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T06:13:50,745 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:13:50,746 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:13:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-06T06:13:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T06:13:50,750 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:13:50,753 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:13:50,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:13:50,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e7e2726d84f4e0d9888a14e3e2c25d5, entries=150, sequenceid=95, filesize=11.7 K 2024-12-06T06:13:50,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 3860e7c2497c4799e160cda168e0086d in 252ms, sequenceid=95, compaction requested=true 2024-12-06T06:13:50,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:50,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:50,761 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:50,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:50,761 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:50,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:50,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:50,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:50,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:50,763 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:50,763 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:50,763 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:13:50,763 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:50,763 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:50,763 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:50,764 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/74727690ca4641d8a26f91279b68b50a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/26932b711a5344c481cc333feb3d995c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4781812c93c5412088f14929e57bdb48] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=37.5 K 2024-12-06T06:13:50,764 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5ac56220f8514825924d12f6fdee3f45, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3af5629cb3fc49eeab9b1cdd0a2b9615, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5875754dafd446cb8aa83c2038d469c3] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=35.3 K 2024-12-06T06:13:50,765 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac56220f8514825924d12f6fdee3f45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733465629016 2024-12-06T06:13:50,765 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74727690ca4641d8a26f91279b68b50a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733465629016 2024-12-06T06:13:50,765 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af5629cb3fc49eeab9b1cdd0a2b9615, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733465629466 2024-12-06T06:13:50,766 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26932b711a5344c481cc333feb3d995c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733465629466 2024-12-06T06:13:50,767 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5875754dafd446cb8aa83c2038d469c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733465629831 2024-12-06T06:13:50,768 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4781812c93c5412088f14929e57bdb48, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733465629831 2024-12-06T06:13:50,803 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:50,804 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8ad1e85278214a2fb0236f038e4072f4 is 50, key is test_row_0/A:col10/1733465630505/Put/seqid=0 2024-12-06T06:13:50,805 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#19 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:50,806 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/7df4809577f0499c9a6406a89dd2d90f is 50, key is test_row_0/B:col10/1733465630505/Put/seqid=0 2024-12-06T06:13:50,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741856_1032 (size=12207) 2024-12-06T06:13:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T06:13:50,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741855_1031 (size=12207) 2024-12-06T06:13:50,863 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/7df4809577f0499c9a6406a89dd2d90f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7df4809577f0499c9a6406a89dd2d90f 2024-12-06T06:13:50,875 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8ad1e85278214a2fb0236f038e4072f4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8ad1e85278214a2fb0236f038e4072f4 2024-12-06T06:13:50,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T06:13:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:50,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4dd7dfac129744b99474900b3e8842b6 is 50, key is test_row_0/A:col10/1733465630877/Put/seqid=0 2024-12-06T06:13:50,893 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 8ad1e85278214a2fb0236f038e4072f4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:50,894 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:50,894 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=13, startTime=1733465630761; duration=0sec 2024-12-06T06:13:50,894 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:50,894 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:50,894 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:50,898 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:50,898 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:50,898 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:50,898 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/483a7bc299374804bbc19b5b93e340e4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8033da6b873945df93089eba6645bba0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e7e2726d84f4e0d9888a14e3e2c25d5] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=35.3 K 2024-12-06T06:13:50,899 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 7df4809577f0499c9a6406a89dd2d90f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:50,899 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:50,899 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=13, startTime=1733465630761; duration=0sec 2024-12-06T06:13:50,899 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:50,899 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:50,901 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483a7bc299374804bbc19b5b93e340e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733465629016 2024-12-06T06:13:50,902 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8033da6b873945df93089eba6645bba0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733465629466 2024-12-06T06:13:50,903 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7e2726d84f4e0d9888a14e3e2c25d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733465629831 2024-12-06T06:13:50,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T06:13:50,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:50,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:50,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:50,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:50,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:50,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741857_1033 (size=14341) 2024-12-06T06:13:50,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4dd7dfac129744b99474900b3e8842b6 2024-12-06T06:13:50,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465690905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465690906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465690910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,940 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465690925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465690927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:50,943 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/5e3bbdb84ba748f194fb682ea401cb7f is 50, key is test_row_0/C:col10/1733465630505/Put/seqid=0 2024-12-06T06:13:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:50,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/c9617a96b11847568ba1aa11966029e5 is 50, key is test_row_0/B:col10/1733465630877/Put/seqid=0 2024-12-06T06:13:50,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741858_1034 (size=12207) 2024-12-06T06:13:50,992 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/5e3bbdb84ba748f194fb682ea401cb7f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/5e3bbdb84ba748f194fb682ea401cb7f 2024-12-06T06:13:51,007 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into 5e3bbdb84ba748f194fb682ea401cb7f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:51,007 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:51,007 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=13, startTime=1733465630762; duration=0sec 2024-12-06T06:13:51,007 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:51,007 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:51,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741859_1035 (size=12001) 2024-12-06T06:13:51,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/c9617a96b11847568ba1aa11966029e5 2024-12-06T06:13:51,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465691036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465691036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465691044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/f47ad99e4ee8477abb4423407f6476c8 is 50, key is test_row_0/C:col10/1733465630877/Put/seqid=0 2024-12-06T06:13:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465691054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465691055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T06:13:51,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T06:13:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:51,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741860_1036 (size=12001) 2024-12-06T06:13:51,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/f47ad99e4ee8477abb4423407f6476c8 2024-12-06T06:13:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4dd7dfac129744b99474900b3e8842b6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dd7dfac129744b99474900b3e8842b6 2024-12-06T06:13:51,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dd7dfac129744b99474900b3e8842b6, entries=200, sequenceid=114, filesize=14.0 K 2024-12-06T06:13:51,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/c9617a96b11847568ba1aa11966029e5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/c9617a96b11847568ba1aa11966029e5 2024-12-06T06:13:51,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/c9617a96b11847568ba1aa11966029e5, entries=150, sequenceid=114, filesize=11.7 K 2024-12-06T06:13:51,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/f47ad99e4ee8477abb4423407f6476c8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f47ad99e4ee8477abb4423407f6476c8 2024-12-06T06:13:51,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f47ad99e4ee8477abb4423407f6476c8, entries=150, sequenceid=114, filesize=11.7 K 2024-12-06T06:13:51,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3860e7c2497c4799e160cda168e0086d in 272ms, sequenceid=114, compaction requested=false 2024-12-06T06:13:51,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:51,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T06:13:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:51,223 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:13:51,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:51,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d93a98ca1a4e428bb920ddefbc3d3878 is 50, key is test_row_0/A:col10/1733465630920/Put/seqid=0 2024-12-06T06:13:51,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:51,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741861_1037 (size=12001) 2024-12-06T06:13:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:51,257 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d93a98ca1a4e428bb920ddefbc3d3878 2024-12-06T06:13:51,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5450c4bb80814d5d83eab816b8721c7c is 50, key is test_row_0/B:col10/1733465630920/Put/seqid=0 2024-12-06T06:13:51,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741862_1038 (size=12001) 2024-12-06T06:13:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465691358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465691358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465691360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T06:13:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465691370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465691366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465691474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465691475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465691484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465691484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465691484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465691685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465691685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465691694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465691694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465691694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:51,729 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5450c4bb80814d5d83eab816b8721c7c 2024-12-06T06:13:51,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/ee2770db77a644faa64dc5cd0f92c672 is 50, key is test_row_0/C:col10/1733465630920/Put/seqid=0 2024-12-06T06:13:51,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741863_1039 (size=12001) 2024-12-06T06:13:51,766 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/ee2770db77a644faa64dc5cd0f92c672 2024-12-06T06:13:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d93a98ca1a4e428bb920ddefbc3d3878 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d93a98ca1a4e428bb920ddefbc3d3878 2024-12-06T06:13:51,788 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d93a98ca1a4e428bb920ddefbc3d3878, entries=150, sequenceid=130, filesize=11.7 K 2024-12-06T06:13:51,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5450c4bb80814d5d83eab816b8721c7c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5450c4bb80814d5d83eab816b8721c7c 2024-12-06T06:13:51,825 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5450c4bb80814d5d83eab816b8721c7c, entries=150, sequenceid=130, filesize=11.7 K 2024-12-06T06:13:51,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/ee2770db77a644faa64dc5cd0f92c672 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ee2770db77a644faa64dc5cd0f92c672 2024-12-06T06:13:51,838 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ee2770db77a644faa64dc5cd0f92c672, entries=150, sequenceid=130, filesize=11.7 K 2024-12-06T06:13:51,840 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3860e7c2497c4799e160cda168e0086d in 617ms, sequenceid=130, compaction requested=true 2024-12-06T06:13:51,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:51,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:51,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-06T06:13:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-06T06:13:51,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-06T06:13:51,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0900 sec 2024-12-06T06:13:51,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1020 sec 2024-12-06T06:13:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T06:13:51,884 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:13:51,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:13:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-06T06:13:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T06:13:51,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:13:51,893 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:13:51,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:13:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T06:13:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:52,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:52,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:52,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/192ffe9db3d5465789582046e27ca263 is 50, key is test_row_0/A:col10/1733465631311/Put/seqid=0 2024-12-06T06:13:52,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465692026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465692030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465692033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465692035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465692037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-06T06:13:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741864_1040 (size=12151) 2024-12-06T06:13:52,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/192ffe9db3d5465789582046e27ca263 2024-12-06T06:13:52,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/f141df63f02c43b7b3b1a1ca2c4d0a34 is 50, key is test_row_0/B:col10/1733465631311/Put/seqid=0 2024-12-06T06:13:52,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741865_1041 (size=12151) 2024-12-06T06:13:52,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/f141df63f02c43b7b3b1a1ca2c4d0a34 2024-12-06T06:13:52,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b9c5d58f9bde4286973bc2a0d883c409 is 50, key is test_row_0/C:col10/1733465631311/Put/seqid=0 2024-12-06T06:13:52,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741866_1042 (size=12151) 2024-12-06T06:13:52,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b9c5d58f9bde4286973bc2a0d883c409 2024-12-06T06:13:52,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465692144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465692146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465692155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465692155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465692158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/192ffe9db3d5465789582046e27ca263 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/192ffe9db3d5465789582046e27ca263 2024-12-06T06:13:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/192ffe9db3d5465789582046e27ca263, entries=150, sequenceid=150, filesize=11.9 K 2024-12-06T06:13:52,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/f141df63f02c43b7b3b1a1ca2c4d0a34 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/f141df63f02c43b7b3b1a1ca2c4d0a34 2024-12-06T06:13:52,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/f141df63f02c43b7b3b1a1ca2c4d0a34, entries=150, sequenceid=150, filesize=11.9 K 2024-12-06T06:13:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b9c5d58f9bde4286973bc2a0d883c409 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b9c5d58f9bde4286973bc2a0d883c409 2024-12-06T06:13:52,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b9c5d58f9bde4286973bc2a0d883c409, entries=150, sequenceid=150, filesize=11.9 K 2024-12-06T06:13:52,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 3860e7c2497c4799e160cda168e0086d in 197ms, sequenceid=150, compaction requested=true 2024-12-06T06:13:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:52,201 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:13:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:52,201 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:13:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:52,203 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:13:52,203 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:13:52,203 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:52,203 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:13:52,203 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,203 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,203 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8ad1e85278214a2fb0236f038e4072f4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dd7dfac129744b99474900b3e8842b6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d93a98ca1a4e428bb920ddefbc3d3878, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/192ffe9db3d5465789582046e27ca263] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=49.5 K 2024-12-06T06:13:52,203 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7df4809577f0499c9a6406a89dd2d90f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/c9617a96b11847568ba1aa11966029e5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5450c4bb80814d5d83eab816b8721c7c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/f141df63f02c43b7b3b1a1ca2c4d0a34] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=47.2 K 2024-12-06T06:13:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T06:13:52,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-06T06:13:52,204 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad1e85278214a2fb0236f038e4072f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733465629831 2024-12-06T06:13:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,205 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:13:52,205 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dd7dfac129744b99474900b3e8842b6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733465630534 2024-12-06T06:13:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:52,205 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d93a98ca1a4e428bb920ddefbc3d3878, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733465630901 2024-12-06T06:13:52,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,206 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 192ffe9db3d5465789582046e27ca263, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733465631311 2024-12-06T06:13:52,208 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df4809577f0499c9a6406a89dd2d90f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733465629831 2024-12-06T06:13:52,210 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting c9617a96b11847568ba1aa11966029e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733465630534 2024-12-06T06:13:52,212 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5450c4bb80814d5d83eab816b8721c7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733465630901 2024-12-06T06:13:52,213 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting f141df63f02c43b7b3b1a1ca2c4d0a34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733465631311 2024-12-06T06:13:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/a68b3fb71520459f9c10beced8846fcb is 50, key is test_row_0/A:col10/1733465632032/Put/seqid=0 2024-12-06T06:13:52,239 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:52,240 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/244ea4819c8e4521b752d1d82de11602 is 50, key is test_row_0/A:col10/1733465631311/Put/seqid=0 2024-12-06T06:13:52,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741867_1043 (size=12151) 2024-12-06T06:13:52,253 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/a68b3fb71520459f9c10beced8846fcb 2024-12-06T06:13:52,254 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#32 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:52,255 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/100a8fc5cbb7474abfa3c0e00e904be4 is 50, key is test_row_0/B:col10/1733465631311/Put/seqid=0 2024-12-06T06:13:52,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/2efd0d52b6c74a1b91b28411b2e69c61 is 50, key is test_row_0/B:col10/1733465632032/Put/seqid=0 2024-12-06T06:13:52,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741868_1044 (size=12493) 2024-12-06T06:13:52,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741870_1046 (size=12151) 2024-12-06T06:13:52,299 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/2efd0d52b6c74a1b91b28411b2e69c61 2024-12-06T06:13:52,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741869_1045 (size=12493) 2024-12-06T06:13:52,305 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/244ea4819c8e4521b752d1d82de11602 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/244ea4819c8e4521b752d1d82de11602 2024-12-06T06:13:52,320 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 244ea4819c8e4521b752d1d82de11602(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:52,320 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:52,320 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=12, startTime=1733465632201; duration=0sec 2024-12-06T06:13:52,321 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:52,321 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:52,321 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:13:52,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/68ef9bd3e4014e7fa494ce171c554e66 is 50, key is test_row_0/C:col10/1733465632032/Put/seqid=0 2024-12-06T06:13:52,326 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:13:52,326 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:52,326 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,328 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/5e3bbdb84ba748f194fb682ea401cb7f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f47ad99e4ee8477abb4423407f6476c8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ee2770db77a644faa64dc5cd0f92c672, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b9c5d58f9bde4286973bc2a0d883c409] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=47.2 K 2024-12-06T06:13:52,329 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e3bbdb84ba748f194fb682ea401cb7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733465629831 2024-12-06T06:13:52,330 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f47ad99e4ee8477abb4423407f6476c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733465630534 2024-12-06T06:13:52,331 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee2770db77a644faa64dc5cd0f92c672, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733465630901 2024-12-06T06:13:52,332 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c5d58f9bde4286973bc2a0d883c409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733465631311 2024-12-06T06:13:52,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741871_1047 (size=12151) 2024-12-06T06:13:52,338 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/68ef9bd3e4014e7fa494ce171c554e66 2024-12-06T06:13:52,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/a68b3fb71520459f9c10beced8846fcb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a68b3fb71520459f9c10beced8846fcb 2024-12-06T06:13:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:52,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:52,361 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a68b3fb71520459f9c10beced8846fcb, entries=150, sequenceid=163, filesize=11.9 K 2024-12-06T06:13:52,361 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#35 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:52,362 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/c67d6287ec2648afa6fb9681f80f0c99 is 50, key is test_row_0/C:col10/1733465631311/Put/seqid=0 2024-12-06T06:13:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/2efd0d52b6c74a1b91b28411b2e69c61 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/2efd0d52b6c74a1b91b28411b2e69c61 2024-12-06T06:13:52,371 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/2efd0d52b6c74a1b91b28411b2e69c61, entries=150, sequenceid=163, filesize=11.9 K 2024-12-06T06:13:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/68ef9bd3e4014e7fa494ce171c554e66 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/68ef9bd3e4014e7fa494ce171c554e66 2024-12-06T06:13:52,381 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/68ef9bd3e4014e7fa494ce171c554e66, entries=150, sequenceid=163, filesize=11.9 K 2024-12-06T06:13:52,382 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=53.67 KB/54960 for 3860e7c2497c4799e160cda168e0086d in 177ms, sequenceid=163, compaction requested=false 2024-12-06T06:13:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-06T06:13:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-06T06:13:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:52,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741872_1048 (size=12493) 2024-12-06T06:13:52,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T06:13:52,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:52,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:52,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:52,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:52,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-06T06:13:52,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 493 msec 2024-12-06T06:13:52,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 506 msec 2024-12-06T06:13:52,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/6c71a0c3dece4687bdee6f2149235b2c is 50, key is test_row_0/A:col10/1733465632383/Put/seqid=0 2024-12-06T06:13:52,406 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/c67d6287ec2648afa6fb9681f80f0c99 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/c67d6287ec2648afa6fb9681f80f0c99 2024-12-06T06:13:52,416 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into c67d6287ec2648afa6fb9681f80f0c99(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:52,416 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:52,416 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=12, startTime=1733465632201; duration=0sec 2024-12-06T06:13:52,417 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:52,417 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:52,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741873_1049 (size=16931) 2024-12-06T06:13:52,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/6c71a0c3dece4687bdee6f2149235b2c 2024-12-06T06:13:52,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/d439f82776f947ef8ebc967a2cd4ea0f is 50, key is test_row_0/B:col10/1733465632383/Put/seqid=0 2024-12-06T06:13:52,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465692436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465692438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741874_1050 (size=12151) 2024-12-06T06:13:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465692441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/d439f82776f947ef8ebc967a2cd4ea0f 2024-12-06T06:13:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465692442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465692443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/d20425bfc169460ab0e324c35b102d73 is 50, key is test_row_0/C:col10/1733465632383/Put/seqid=0 2024-12-06T06:13:52,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741875_1051 (size=12151) 2024-12-06T06:13:52,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/d20425bfc169460ab0e324c35b102d73 2024-12-06T06:13:52,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/6c71a0c3dece4687bdee6f2149235b2c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/6c71a0c3dece4687bdee6f2149235b2c 2024-12-06T06:13:52,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/6c71a0c3dece4687bdee6f2149235b2c, entries=250, sequenceid=176, filesize=16.5 K 2024-12-06T06:13:52,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/d439f82776f947ef8ebc967a2cd4ea0f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/d439f82776f947ef8ebc967a2cd4ea0f 2024-12-06T06:13:52,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/d439f82776f947ef8ebc967a2cd4ea0f, entries=150, sequenceid=176, filesize=11.9 K 2024-12-06T06:13:52,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/d20425bfc169460ab0e324c35b102d73 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/d20425bfc169460ab0e324c35b102d73 2024-12-06T06:13:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T06:13:52,514 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:13:52,515 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-06T06:13:52,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:13:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T06:13:52,524 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:13:52,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:13:52,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/d20425bfc169460ab0e324c35b102d73, entries=150, sequenceid=176, filesize=11.9 K 2024-12-06T06:13:52,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3860e7c2497c4799e160cda168e0086d in 149ms, sequenceid=176, compaction requested=true 2024-12-06T06:13:52,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:52,535 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:52,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:52,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:52,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:52,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:52,537 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:13:52,537 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:52,537 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,537 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/244ea4819c8e4521b752d1d82de11602, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a68b3fb71520459f9c10beced8846fcb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/6c71a0c3dece4687bdee6f2149235b2c] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=40.6 K 2024-12-06T06:13:52,538 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244ea4819c8e4521b752d1d82de11602, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733465631311 2024-12-06T06:13:52,538 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a68b3fb71520459f9c10beced8846fcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733465632011 2024-12-06T06:13:52,539 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c71a0c3dece4687bdee6f2149235b2c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733465632363 2024-12-06T06:13:52,560 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:52,561 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8a7604e939c641f5afaa4a54677ab410 is 50, key is test_row_0/A:col10/1733465632383/Put/seqid=0 2024-12-06T06:13:52,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465632431, max timestamp=1733465632562Num uniques 859; 2024-12-06T06:13:52,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741876_1052 (size=12595) 2024-12-06T06:13:52,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465632431, max timestamp=1733465632565Num uniques 859; 2024-12-06T06:13:52,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:13:52,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:13:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:52,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=71.56 KB heapSize=204.02 KB 2024-12-06T06:13:52,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:52,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:52,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:52,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:52,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:52,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:52,584 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8a7604e939c641f5afaa4a54677ab410 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8a7604e939c641f5afaa4a54677ab410 2024-12-06T06:13:52,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/7657638d83b94390b15d547bc552d6d9 is 50, key is test_row_0/A:col10/1733465632565/Put/seqid=0 2024-12-06T06:13:52,603 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 8a7604e939c641f5afaa4a54677ab410(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:52,603 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:52,603 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=13, startTime=1733465632534; duration=0sec 2024-12-06T06:13:52,604 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:13:52,604 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:52,604 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-12-06T06:13:52,606 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-06T06:13:52,606 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-06T06:13:52,606 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. because compaction request was cancelled 2024-12-06T06:13:52,607 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:52,607 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:52,608 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:52,608 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:52,608 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,609 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/c67d6287ec2648afa6fb9681f80f0c99, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/68ef9bd3e4014e7fa494ce171c554e66, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/d20425bfc169460ab0e324c35b102d73] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=35.9 K 2024-12-06T06:13:52,609 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c67d6287ec2648afa6fb9681f80f0c99, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733465631311 2024-12-06T06:13:52,610 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ef9bd3e4014e7fa494ce171c554e66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733465632011 2024-12-06T06:13:52,611 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d20425bfc169460ab0e324c35b102d73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733465632363 2024-12-06T06:13:52,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741877_1053 (size=12151) 2024-12-06T06:13:52,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/7657638d83b94390b15d547bc552d6d9 2024-12-06T06:13:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T06:13:52,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465692627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465692630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,642 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:52,643 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b619c297c5b14700ba724bfc73852680 is 50, key is test_row_0/C:col10/1733465632383/Put/seqid=0 2024-12-06T06:13:52,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465692636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465692638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/18e1155fb78d473a862a79a6e03602bb is 50, key is test_row_0/B:col10/1733465632565/Put/seqid=0 2024-12-06T06:13:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465692639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741879_1055 (size=12151) 2024-12-06T06:13:52,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741878_1054 (size=12595) 2024-12-06T06:13:52,702 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b619c297c5b14700ba724bfc73852680 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b619c297c5b14700ba724bfc73852680 2024-12-06T06:13:52,715 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/100a8fc5cbb7474abfa3c0e00e904be4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/100a8fc5cbb7474abfa3c0e00e904be4 2024-12-06T06:13:52,719 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into b619c297c5b14700ba724bfc73852680(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:52,719 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:52,719 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=13, startTime=1733465632537; duration=0sec 2024-12-06T06:13:52,719 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:52,719 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:52,730 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 100a8fc5cbb7474abfa3c0e00e904be4(size=12.2 K), total size for store is 35.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:52,730 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:52,730 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=12, startTime=1733465632201; duration=0sec 2024-12-06T06:13:52,730 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:52,730 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:52,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465692744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465692745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465692754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465692754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465692754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T06:13:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465692954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465692954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465692963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465692974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465692974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:52,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:52,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/18e1155fb78d473a862a79a6e03602bb 2024-12-06T06:13:53,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1e742c8dd10e4cf7aa0ca210a63eb33d is 50, key is test_row_0/C:col10/1733465632565/Put/seqid=0 2024-12-06T06:13:53,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741880_1056 (size=12151) 2024-12-06T06:13:53,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1e742c8dd10e4cf7aa0ca210a63eb33d 2024-12-06T06:13:53,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/7657638d83b94390b15d547bc552d6d9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7657638d83b94390b15d547bc552d6d9 2024-12-06T06:13:53,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7657638d83b94390b15d547bc552d6d9, entries=150, sequenceid=200, filesize=11.9 K 2024-12-06T06:13:53,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/18e1155fb78d473a862a79a6e03602bb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/18e1155fb78d473a862a79a6e03602bb 2024-12-06T06:13:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T06:13:53,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/18e1155fb78d473a862a79a6e03602bb, entries=150, sequenceid=200, filesize=11.9 K 2024-12-06T06:13:53,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1e742c8dd10e4cf7aa0ca210a63eb33d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e742c8dd10e4cf7aa0ca210a63eb33d 2024-12-06T06:13:53,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e742c8dd10e4cf7aa0ca210a63eb33d, entries=150, sequenceid=200, filesize=11.9 K 2024-12-06T06:13:53,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=107.34 KB/109920 for 3860e7c2497c4799e160cda168e0086d in 605ms, sequenceid=200, compaction requested=true 2024-12-06T06:13:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:53,178 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-06T06:13:53,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:53,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:53,179 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:13:53,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:53,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:53,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:53,180 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-06T06:13:53,180 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-06T06:13:53,180 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. because compaction request was cancelled 2024-12-06T06:13:53,180 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:53,180 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-06T06:13:53,182 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-06T06:13:53,182 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-06T06:13:53,182 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. because compaction request was cancelled 2024-12-06T06:13:53,182 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:53,182 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:13:53,182 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:13:53,182 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,183 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/100a8fc5cbb7474abfa3c0e00e904be4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/2efd0d52b6c74a1b91b28411b2e69c61, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/d439f82776f947ef8ebc967a2cd4ea0f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/18e1155fb78d473a862a79a6e03602bb] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=47.8 K 2024-12-06T06:13:53,183 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 100a8fc5cbb7474abfa3c0e00e904be4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733465631311 2024-12-06T06:13:53,184 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efd0d52b6c74a1b91b28411b2e69c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733465632011 2024-12-06T06:13:53,184 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d439f82776f947ef8ebc967a2cd4ea0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733465632363 2024-12-06T06:13:53,185 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 18e1155fb78d473a862a79a6e03602bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733465632438 2024-12-06T06:13:53,201 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:53,201 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/70e4f0d1026746f7a1170fa443b5a7b4 is 50, key is test_row_0/B:col10/1733465632565/Put/seqid=0 2024-12-06T06:13:53,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741881_1057 (size=12629) 2024-12-06T06:13:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:53,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:13:53,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465632604, max timestamp=1733465633277Num uniques 859; 2024-12-06T06:13:53,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:53,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465632604, max timestamp=1733465633277Num uniques 859; 2024-12-06T06:13:53,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:53,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/ea99e8af47da41ed84943f4177029237 is 50, key is test_row_0/A:col10/1733465632630/Put/seqid=0 2024-12-06T06:13:53,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465693320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465693320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465693322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465693327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465693328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741882_1058 (size=14541) 2024-12-06T06:13:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465693434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465693434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465693434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465693435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465693435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465693644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465693645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465693645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465693645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465693644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,655 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/70e4f0d1026746f7a1170fa443b5a7b4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/70e4f0d1026746f7a1170fa443b5a7b4 2024-12-06T06:13:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T06:13:53,666 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 70e4f0d1026746f7a1170fa443b5a7b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:53,666 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:53,666 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=12, startTime=1733465633179; duration=0sec 2024-12-06T06:13:53,666 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:53,666 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:53,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/ea99e8af47da41ed84943f4177029237 2024-12-06T06:13:53,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/231f4db8f91b4ecb8b1410b5f49f2805 is 50, key is test_row_0/B:col10/1733465632630/Put/seqid=0 2024-12-06T06:13:53,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741883_1059 (size=12151) 2024-12-06T06:13:53,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:53,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/231f4db8f91b4ecb8b1410b5f49f2805 2024-12-06T06:13:53,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/e0fe639a36414e20ba9b91a463a1c4b3 is 50, key is test_row_0/C:col10/1733465632630/Put/seqid=0 2024-12-06T06:13:53,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741884_1060 (size=12151) 2024-12-06T06:13:53,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/e0fe639a36414e20ba9b91a463a1c4b3 2024-12-06T06:13:53,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/ea99e8af47da41ed84943f4177029237 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ea99e8af47da41ed84943f4177029237 2024-12-06T06:13:53,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ea99e8af47da41ed84943f4177029237, entries=200, sequenceid=224, filesize=14.2 K 2024-12-06T06:13:53,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/231f4db8f91b4ecb8b1410b5f49f2805 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/231f4db8f91b4ecb8b1410b5f49f2805 2024-12-06T06:13:53,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/231f4db8f91b4ecb8b1410b5f49f2805, entries=150, sequenceid=224, filesize=11.9 K 2024-12-06T06:13:53,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/e0fe639a36414e20ba9b91a463a1c4b3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e0fe639a36414e20ba9b91a463a1c4b3 2024-12-06T06:13:53,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e0fe639a36414e20ba9b91a463a1c4b3, entries=150, sequenceid=224, filesize=11.9 K 2024-12-06T06:13:53,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.62 KB/64120, heapSize ~174.28 KB/178464, currentSize=114.05 KB/116790 for 3860e7c2497c4799e160cda168e0086d in 632ms, sequenceid=224, compaction requested=true 2024-12-06T06:13:53,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:53,909 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:53,911 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:53,911 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:53,911 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,911 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8a7604e939c641f5afaa4a54677ab410, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7657638d83b94390b15d547bc552d6d9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ea99e8af47da41ed84943f4177029237] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=38.4 K 2024-12-06T06:13:53,912 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7604e939c641f5afaa4a54677ab410, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733465632363 2024-12-06T06:13:53,912 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7657638d83b94390b15d547bc552d6d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733465632438 2024-12-06T06:13:53,913 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea99e8af47da41ed84943f4177029237, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733465632630 2024-12-06T06:13:53,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:53,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:53,925 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-06T06:13:53,926 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-06T06:13:53,926 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-06T06:13:53,926 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. because compaction request was cancelled 2024-12-06T06:13:53,926 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:53,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:53,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:53,926 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:53,927 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:53,927 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:53,928 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,928 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b619c297c5b14700ba724bfc73852680, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e742c8dd10e4cf7aa0ca210a63eb33d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e0fe639a36414e20ba9b91a463a1c4b3] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=36.0 K 2024-12-06T06:13:53,929 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b619c297c5b14700ba724bfc73852680, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733465632363 2024-12-06T06:13:53,929 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:53,930 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e742c8dd10e4cf7aa0ca210a63eb33d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733465632438 2024-12-06T06:13:53,930 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/93d6c6b25c3e491b8cb8afabd769d370 is 50, key is test_row_0/A:col10/1733465632630/Put/seqid=0 2024-12-06T06:13:53,931 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e0fe639a36414e20ba9b91a463a1c4b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733465632630 2024-12-06T06:13:53,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T06:13:53,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:53,937 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:13:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:53,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:53,945 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:53,946 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/64d14fc0a8ec4256be5b75322bf47e2b is 50, key is test_row_0/C:col10/1733465632630/Put/seqid=0 2024-12-06T06:13:53,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2 is 50, key is test_row_0/A:col10/1733465633317/Put/seqid=0 2024-12-06T06:13:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:53,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741885_1061 (size=12697) 2024-12-06T06:13:53,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465693986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:53,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465693986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,002 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/93d6c6b25c3e491b8cb8afabd769d370 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/93d6c6b25c3e491b8cb8afabd769d370 2024-12-06T06:13:54,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465693993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465693993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465693995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,011 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 93d6c6b25c3e491b8cb8afabd769d370(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:54,011 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:54,011 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=13, startTime=1733465633909; duration=0sec 2024-12-06T06:13:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,012 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,012 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741886_1062 (size=12697) 2024-12-06T06:13:54,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741887_1063 (size=12151) 2024-12-06T06:13:54,027 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/64d14fc0a8ec4256be5b75322bf47e2b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/64d14fc0a8ec4256be5b75322bf47e2b 2024-12-06T06:13:54,042 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into 64d14fc0a8ec4256be5b75322bf47e2b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:54,042 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:54,042 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=13, startTime=1733465633926; duration=0sec 2024-12-06T06:13:54,042 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:54,042 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465694105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465694105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465694115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465694116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465694116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465694314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465694314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465694324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465694324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465694324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,420 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2 2024-12-06T06:13:54,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/fe49e272b7804db0805ec0d610cba219 is 50, key is test_row_0/B:col10/1733465633317/Put/seqid=0 2024-12-06T06:13:54,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741888_1064 (size=12151) 2024-12-06T06:13:54,445 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/fe49e272b7804db0805ec0d610cba219 2024-12-06T06:13:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/90d7d0bf77384c62bc84aea4df30c460 is 50, key is test_row_0/C:col10/1733465633317/Put/seqid=0 2024-12-06T06:13:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741889_1065 (size=12151) 2024-12-06T06:13:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465694623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465694624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465694634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465694634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465694634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T06:13:54,871 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/90d7d0bf77384c62bc84aea4df30c460 2024-12-06T06:13:54,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2 2024-12-06T06:13:54,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2, entries=150, sequenceid=245, filesize=11.9 K 2024-12-06T06:13:54,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/fe49e272b7804db0805ec0d610cba219 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/fe49e272b7804db0805ec0d610cba219 2024-12-06T06:13:54,898 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/fe49e272b7804db0805ec0d610cba219, entries=150, sequenceid=245, filesize=11.9 K 2024-12-06T06:13:54,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/90d7d0bf77384c62bc84aea4df30c460 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90d7d0bf77384c62bc84aea4df30c460 2024-12-06T06:13:54,910 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90d7d0bf77384c62bc84aea4df30c460, entries=150, sequenceid=245, filesize=11.9 K 2024-12-06T06:13:54,912 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 3860e7c2497c4799e160cda168e0086d in 975ms, sequenceid=245, compaction requested=true 2024-12-06T06:13:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-06T06:13:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-06T06:13:54,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-06T06:13:54,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3920 sec 2024-12-06T06:13:54,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.4070 sec 2024-12-06T06:13:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:55,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T06:13:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:55,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/a9633258a7db49b3b3d21d8230a1b0b1 is 50, key is test_row_0/A:col10/1733465633970/Put/seqid=0 2024-12-06T06:13:55,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741890_1066 (size=14641) 2024-12-06T06:13:55,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/a9633258a7db49b3b3d21d8230a1b0b1 2024-12-06T06:13:55,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/7396d3d1c13a436d9fa78753ee2c1a9f is 50, key is test_row_0/B:col10/1733465633970/Put/seqid=0 2024-12-06T06:13:55,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465695179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465695184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465695195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465695198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741891_1067 (size=12201) 2024-12-06T06:13:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465695201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/7396d3d1c13a436d9fa78753ee2c1a9f 2024-12-06T06:13:55,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/54b3a749018447c7835b4d860a72453b is 50, key is test_row_0/C:col10/1733465633970/Put/seqid=0 2024-12-06T06:13:55,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741892_1068 (size=12201) 2024-12-06T06:13:55,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/54b3a749018447c7835b4d860a72453b 2024-12-06T06:13:55,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/a9633258a7db49b3b3d21d8230a1b0b1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a9633258a7db49b3b3d21d8230a1b0b1 2024-12-06T06:13:55,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a9633258a7db49b3b3d21d8230a1b0b1, entries=200, sequenceid=261, filesize=14.3 K 2024-12-06T06:13:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/7396d3d1c13a436d9fa78753ee2c1a9f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7396d3d1c13a436d9fa78753ee2c1a9f 2024-12-06T06:13:55,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7396d3d1c13a436d9fa78753ee2c1a9f, entries=150, sequenceid=261, filesize=11.9 K 2024-12-06T06:13:55,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/54b3a749018447c7835b4d860a72453b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/54b3a749018447c7835b4d860a72453b 2024-12-06T06:13:55,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/54b3a749018447c7835b4d860a72453b, entries=150, sequenceid=261, filesize=11.9 K 2024-12-06T06:13:55,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 3860e7c2497c4799e160cda168e0086d in 142ms, sequenceid=261, compaction requested=true 2024-12-06T06:13:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:55,281 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:55,282 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:13:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:55,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:55,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:55,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:55,283 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39489 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:55,284 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:55,284 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:55,284 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:13:55,284 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:13:55,284 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/93d6c6b25c3e491b8cb8afabd769d370, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a9633258a7db49b3b3d21d8230a1b0b1] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=38.6 K 2024-12-06T06:13:55,284 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:55,284 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d6c6b25c3e491b8cb8afabd769d370, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733465632630 2024-12-06T06:13:55,284 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/70e4f0d1026746f7a1170fa443b5a7b4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/231f4db8f91b4ecb8b1410b5f49f2805, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/fe49e272b7804db0805ec0d610cba219, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7396d3d1c13a436d9fa78753ee2c1a9f] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=48.0 K 2024-12-06T06:13:55,285 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf3bd24e7daa4747a0f3bc5b6d1d27a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733465633317 2024-12-06T06:13:55,287 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e4f0d1026746f7a1170fa443b5a7b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733465632438 2024-12-06T06:13:55,287 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9633258a7db49b3b3d21d8230a1b0b1, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465633970 2024-12-06T06:13:55,288 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 231f4db8f91b4ecb8b1410b5f49f2805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733465632630 2024-12-06T06:13:55,288 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting fe49e272b7804db0805ec0d610cba219, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733465633317 2024-12-06T06:13:55,289 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7396d3d1c13a436d9fa78753ee2c1a9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465633970 2024-12-06T06:13:55,306 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:55,307 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/e3a326a45d7b4b209da9c3810a9efcff is 50, key is test_row_0/A:col10/1733465633970/Put/seqid=0 2024-12-06T06:13:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:55,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:13:55,311 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#57 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:55,312 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/0c4d272675dd4b26b4adc3c730dbfa5d is 50, key is test_row_0/B:col10/1733465633970/Put/seqid=0 2024-12-06T06:13:55,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465635190, max timestamp=1733465635309Num uniques 859; 2024-12-06T06:13:55,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:13:55,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465635190, max timestamp=1733465635309Num uniques 859; 2024-12-06T06:13:55,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465635190, max timestamp=1733465635309Num uniques 859; 2024-12-06T06:13:55,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:13:55,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:13:55,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:55,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:55,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:55,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:55,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:55,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:55,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741893_1069 (size=12849) 2024-12-06T06:13:55,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/cd946af21f714263a77e8df7c2176a05 is 50, key is test_row_0/A:col10/1733465635190/Put/seqid=0 2024-12-06T06:13:55,342 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/e3a326a45d7b4b209da9c3810a9efcff as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e3a326a45d7b4b209da9c3810a9efcff 2024-12-06T06:13:55,357 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into e3a326a45d7b4b209da9c3810a9efcff(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:55,358 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:55,358 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=13, startTime=1733465635281; duration=0sec 2024-12-06T06:13:55,358 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:55,358 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:55,358 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:55,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741894_1070 (size=12815) 2024-12-06T06:13:55,360 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:55,360 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:55,361 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:55,361 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/64d14fc0a8ec4256be5b75322bf47e2b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90d7d0bf77384c62bc84aea4df30c460, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/54b3a749018447c7835b4d860a72453b] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=36.2 K 2024-12-06T06:13:55,361 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d14fc0a8ec4256be5b75322bf47e2b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733465632630 2024-12-06T06:13:55,363 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d7d0bf77384c62bc84aea4df30c460, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733465633317 2024-12-06T06:13:55,364 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b3a749018447c7835b4d860a72453b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465633970 2024-12-06T06:13:55,367 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/0c4d272675dd4b26b4adc3c730dbfa5d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/0c4d272675dd4b26b4adc3c730dbfa5d 2024-12-06T06:13:55,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465635351, max timestamp=1733465635369Num uniques 859; 2024-12-06T06:13:55,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:55,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465635351, max timestamp=1733465635369Num uniques 859; 2024-12-06T06:13:55,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465635351, max timestamp=1733465635369Num uniques 859; 2024-12-06T06:13:55,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:55,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:55,377 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 0c4d272675dd4b26b4adc3c730dbfa5d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:55,377 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:55,377 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=12, startTime=1733465635282; duration=0sec 2024-12-06T06:13:55,378 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:55,378 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:55,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741895_1071 (size=12301) 2024-12-06T06:13:55,410 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:55,411 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/f75d0f4fa5c7481295b89b19bc437750 is 50, key is test_row_0/C:col10/1733465633970/Put/seqid=0 2024-12-06T06:13:55,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741896_1072 (size=12849) 2024-12-06T06:13:55,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=468, cellSize=21428, totalHeapSize=48912, min timestamp=1733465635391, max timestamp=1733465635422Num uniques 1068; 2024-12-06T06:13:55,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=21428, suffix heap size=139040,new segment heap size=48912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=468 2024-12-06T06:13:55,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=468, cellSize=21428, totalHeapSize=48912, min timestamp=1733465635391, max timestamp=1733465635422Num uniques 1068; 2024-12-06T06:13:55,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=21428, suffix heap size=139040,new segment heap size=48912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=468 2024-12-06T06:13:55,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=468, cellSize=21428, totalHeapSize=48912, min timestamp=1733465635391, max timestamp=1733465635422Num uniques 1068; 2024-12-06T06:13:55,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=21428, suffix heap size=139040,new segment heap size=48912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=468 2024-12-06T06:13:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465695470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465695472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465695466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465695475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465695476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465695585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465695585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465695586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465695586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465695587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/cd946af21f714263a77e8df7c2176a05 2024-12-06T06:13:55,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465695805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465695806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465695806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465695807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465695808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:55,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:55,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:55,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/03ddffeb5d184835bb018cae3f7190f4 is 50, key is test_row_0/B:col10/1733465635190/Put/seqid=0 2024-12-06T06:13:55,836 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/f75d0f4fa5c7481295b89b19bc437750 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f75d0f4fa5c7481295b89b19bc437750 2024-12-06T06:13:55,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741897_1073 (size=12301) 2024-12-06T06:13:55,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/03ddffeb5d184835bb018cae3f7190f4 2024-12-06T06:13:55,852 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into f75d0f4fa5c7481295b89b19bc437750(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:55,852 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:55,852 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=13, startTime=1733465635283; duration=0sec 2024-12-06T06:13:55,852 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:55,852 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:55,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/cff709e8108944ff92ae70e10f3e43a1 is 50, key is test_row_0/C:col10/1733465635190/Put/seqid=0 2024-12-06T06:13:55,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741898_1074 (size=12301) 2024-12-06T06:13:55,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/cff709e8108944ff92ae70e10f3e43a1 2024-12-06T06:13:55,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/cd946af21f714263a77e8df7c2176a05 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd946af21f714263a77e8df7c2176a05 2024-12-06T06:13:55,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd946af21f714263a77e8df7c2176a05, entries=150, sequenceid=282, filesize=12.0 K 2024-12-06T06:13:55,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/03ddffeb5d184835bb018cae3f7190f4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ddffeb5d184835bb018cae3f7190f4 2024-12-06T06:13:55,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ddffeb5d184835bb018cae3f7190f4, entries=150, sequenceid=282, filesize=12.0 K 2024-12-06T06:13:55,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/cff709e8108944ff92ae70e10f3e43a1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cff709e8108944ff92ae70e10f3e43a1 2024-12-06T06:13:55,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cff709e8108944ff92ae70e10f3e43a1, entries=150, sequenceid=282, filesize=12.0 K 2024-12-06T06:13:55,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=174.43 KB/178620 for 3860e7c2497c4799e160cda168e0086d in 618ms, sequenceid=282, compaction requested=false 2024-12-06T06:13:55,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:56,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=181.14 KB heapSize=496.17 KB 2024-12-06T06:13:56,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:56,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:13:56,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:56,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:13:56,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:56,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:13:56,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465696115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465696119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465696125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465696126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465696128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/e53aeabf26e94efeb725a66f3ad5514f is 50, key is test_row_0/A:col10/1733465635471/Put/seqid=0 2024-12-06T06:13:56,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741899_1075 (size=12301) 2024-12-06T06:13:56,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/e53aeabf26e94efeb725a66f3ad5514f 2024-12-06T06:13:56,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/a223675fbfb2416db1ad29df1a7fcf10 is 50, key is test_row_0/B:col10/1733465635471/Put/seqid=0 2024-12-06T06:13:56,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741900_1076 (size=12301) 2024-12-06T06:13:56,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/a223675fbfb2416db1ad29df1a7fcf10 2024-12-06T06:13:56,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/8b65e4bb3bbf4b408dc60b21386c7ff0 is 50, key is test_row_0/C:col10/1733465635471/Put/seqid=0 2024-12-06T06:13:56,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741901_1077 (size=12301) 2024-12-06T06:13:56,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/8b65e4bb3bbf4b408dc60b21386c7ff0 2024-12-06T06:13:56,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/e53aeabf26e94efeb725a66f3ad5514f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e53aeabf26e94efeb725a66f3ad5514f 2024-12-06T06:13:56,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e53aeabf26e94efeb725a66f3ad5514f, entries=150, sequenceid=340, filesize=12.0 K 2024-12-06T06:13:56,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/a223675fbfb2416db1ad29df1a7fcf10 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a223675fbfb2416db1ad29df1a7fcf10 2024-12-06T06:13:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465696235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465696235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465696235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a223675fbfb2416db1ad29df1a7fcf10, entries=150, sequenceid=340, filesize=12.0 K 2024-12-06T06:13:56,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/8b65e4bb3bbf4b408dc60b21386c7ff0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8b65e4bb3bbf4b408dc60b21386c7ff0 2024-12-06T06:13:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465696246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8b65e4bb3bbf4b408dc60b21386c7ff0, entries=150, sequenceid=340, filesize=12.0 K 2024-12-06T06:13:56,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~496.13 KB/508032, currentSize=20.13 KB/20610 for 3860e7c2497c4799e160cda168e0086d in 133ms, sequenceid=340, compaction requested=true 2024-12-06T06:13:56,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:56,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:56,252 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:56,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:56,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:56,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:56,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:56,252 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:56,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:56,253 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:56,253 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:13:56,254 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:56,254 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/0c4d272675dd4b26b4adc3c730dbfa5d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ddffeb5d184835bb018cae3f7190f4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a223675fbfb2416db1ad29df1a7fcf10] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=36.5 K 2024-12-06T06:13:56,254 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c4d272675dd4b26b4adc3c730dbfa5d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465633970 2024-12-06T06:13:56,255 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ddffeb5d184835bb018cae3f7190f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733465635190 2024-12-06T06:13:56,256 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a223675fbfb2416db1ad29df1a7fcf10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733465635469 2024-12-06T06:13:56,258 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:56,258 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:56,258 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:56,259 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e3a326a45d7b4b209da9c3810a9efcff, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd946af21f714263a77e8df7c2176a05, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e53aeabf26e94efeb725a66f3ad5514f] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=36.6 K 2024-12-06T06:13:56,260 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3a326a45d7b4b209da9c3810a9efcff, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465633970 2024-12-06T06:13:56,261 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd946af21f714263a77e8df7c2176a05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733465635190 2024-12-06T06:13:56,262 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e53aeabf26e94efeb725a66f3ad5514f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733465635469 2024-12-06T06:13:56,275 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:56,275 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/80169a8dbb5a4b18a2af081e6a2577e0 is 50, key is test_row_0/B:col10/1733465635471/Put/seqid=0 2024-12-06T06:13:56,290 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:56,291 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/5db8ea3d7d2144b69430a7ad24ae22a1 is 50, key is test_row_0/A:col10/1733465635471/Put/seqid=0 2024-12-06T06:13:56,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741902_1078 (size=13017) 2024-12-06T06:13:56,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741903_1079 (size=13051) 2024-12-06T06:13:56,328 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/5db8ea3d7d2144b69430a7ad24ae22a1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5db8ea3d7d2144b69430a7ad24ae22a1 2024-12-06T06:13:56,338 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 5db8ea3d7d2144b69430a7ad24ae22a1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:56,338 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:56,339 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=13, startTime=1733465636252; duration=0sec 2024-12-06T06:13:56,339 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:56,339 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:56,339 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:56,341 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:56,341 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:56,341 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:56,341 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f75d0f4fa5c7481295b89b19bc437750, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cff709e8108944ff92ae70e10f3e43a1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8b65e4bb3bbf4b408dc60b21386c7ff0] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=36.6 K 2024-12-06T06:13:56,342 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f75d0f4fa5c7481295b89b19bc437750, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465633970 2024-12-06T06:13:56,342 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cff709e8108944ff92ae70e10f3e43a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733465635190 2024-12-06T06:13:56,343 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b65e4bb3bbf4b408dc60b21386c7ff0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733465635469 2024-12-06T06:13:56,353 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:56,354 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/9da458e7fcca4033a79e8bedca558948 is 50, key is test_row_0/C:col10/1733465635471/Put/seqid=0 2024-12-06T06:13:56,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741904_1080 (size=13051) 2024-12-06T06:13:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:56,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:13:56,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:56,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:56,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:56,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:56,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:56,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:56,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/02e9ec369f6d43728c2032fb374579b1 is 50, key is test_row_0/A:col10/1733465636450/Put/seqid=0 2024-12-06T06:13:56,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741905_1081 (size=14741) 2024-12-06T06:13:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465636508, max timestamp=1733465636527Num uniques 859; 2024-12-06T06:13:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465636508, max timestamp=1733465636527Num uniques 859; 2024-12-06T06:13:56,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:56,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733465636508, max timestamp=1733465636525Num uniques 859; 2024-12-06T06:13:56,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T06:13:56,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465696576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465696577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465696582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465696583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465696634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T06:13:56,694 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:13:56,695 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:13:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-06T06:13:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T06:13:56,698 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:13:56,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465696694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465696695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,699 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:13:56,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:13:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465696695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465696695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,714 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/80169a8dbb5a4b18a2af081e6a2577e0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/80169a8dbb5a4b18a2af081e6a2577e0 2024-12-06T06:13:56,722 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 80169a8dbb5a4b18a2af081e6a2577e0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:56,722 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:56,722 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=13, startTime=1733465636252; duration=0sec 2024-12-06T06:13:56,723 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:56,723 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:56,774 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/9da458e7fcca4033a79e8bedca558948 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9da458e7fcca4033a79e8bedca558948 2024-12-06T06:13:56,781 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into 9da458e7fcca4033a79e8bedca558948(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:56,781 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:56,781 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=13, startTime=1733465636252; duration=0sec 2024-12-06T06:13:56,781 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:56,781 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T06:13:56,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T06:13:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:56,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:56,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/02e9ec369f6d43728c2032fb374579b1 2024-12-06T06:13:56,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/8db2141e77d34ec4ad6198aa68c86404 is 50, key is test_row_0/B:col10/1733465636450/Put/seqid=0 2024-12-06T06:13:56,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741906_1082 (size=12301) 2024-12-06T06:13:56,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/8db2141e77d34ec4ad6198aa68c86404 2024-12-06T06:13:56,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/846c3019add040b9bdf0d0380ecab78b is 50, key is test_row_0/C:col10/1733465636450/Put/seqid=0 2024-12-06T06:13:56,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465696904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465696904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465696905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465696906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:56,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741907_1083 (size=12301) 2024-12-06T06:13:57,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T06:13:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T06:13:57,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T06:13:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465697224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465697224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465697224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465697224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/846c3019add040b9bdf0d0380ecab78b 2024-12-06T06:13:57,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T06:13:57,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/02e9ec369f6d43728c2032fb374579b1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/02e9ec369f6d43728c2032fb374579b1 2024-12-06T06:13:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T06:13:57,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/02e9ec369f6d43728c2032fb374579b1, entries=200, sequenceid=351, filesize=14.4 K 2024-12-06T06:13:57,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/8db2141e77d34ec4ad6198aa68c86404 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/8db2141e77d34ec4ad6198aa68c86404 2024-12-06T06:13:57,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/8db2141e77d34ec4ad6198aa68c86404, entries=150, sequenceid=351, filesize=12.0 K 2024-12-06T06:13:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/846c3019add040b9bdf0d0380ecab78b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/846c3019add040b9bdf0d0380ecab78b 2024-12-06T06:13:57,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/846c3019add040b9bdf0d0380ecab78b, entries=150, sequenceid=351, filesize=12.0 K 2024-12-06T06:13:57,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.55 KB/134705 for 3860e7c2497c4799e160cda168e0086d in 895ms, sequenceid=351, compaction requested=false 2024-12-06T06:13:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:57,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T06:13:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,474 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-12-06T06:13:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:57,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:57,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:57,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:57,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/f3113c5ea43448c1ad35c04ba9117924 is 50, key is test_row_0/A:col10/1733465636581/Put/seqid=0 2024-12-06T06:13:57,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741908_1084 (size=12301) 2024-12-06T06:13:57,495 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/f3113c5ea43448c1ad35c04ba9117924 2024-12-06T06:13:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/a1756085530745d5b7076dde374cb9cc is 50, key is test_row_0/B:col10/1733465636581/Put/seqid=0 2024-12-06T06:13:57,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741909_1085 (size=12301) 2024-12-06T06:13:57,520 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/a1756085530745d5b7076dde374cb9cc 2024-12-06T06:13:57,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/66a76128591d4ae6bc327de0e1c209fb is 50, key is test_row_0/C:col10/1733465636581/Put/seqid=0 2024-12-06T06:13:57,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741910_1086 (size=12301) 2024-12-06T06:13:57,554 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/66a76128591d4ae6bc327de0e1c209fb 2024-12-06T06:13:57,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/f3113c5ea43448c1ad35c04ba9117924 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f3113c5ea43448c1ad35c04ba9117924 2024-12-06T06:13:57,579 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f3113c5ea43448c1ad35c04ba9117924, entries=150, sequenceid=389, filesize=12.0 K 2024-12-06T06:13:57,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/a1756085530745d5b7076dde374cb9cc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a1756085530745d5b7076dde374cb9cc 2024-12-06T06:13:57,587 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a1756085530745d5b7076dde374cb9cc, entries=150, sequenceid=389, filesize=12.0 K 2024-12-06T06:13:57,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/66a76128591d4ae6bc327de0e1c209fb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/66a76128591d4ae6bc327de0e1c209fb 2024-12-06T06:13:57,596 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/66a76128591d4ae6bc327de0e1c209fb, entries=150, sequenceid=389, filesize=12.0 K 2024-12-06T06:13:57,598 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=0 B/0 for 3860e7c2497c4799e160cda168e0086d in 123ms, sequenceid=389, compaction requested=true 2024-12-06T06:13:57,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:57,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-06T06:13:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-06T06:13:57,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-06T06:13:57,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 901 msec 2024-12-06T06:13:57,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 911 msec 2024-12-06T06:13:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:57,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:57,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8fc31f5c48e64eb898ced3b9f9a765ba is 50, key is test_row_0/A:col10/1733465637671/Put/seqid=0 2024-12-06T06:13:57,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741911_1087 (size=12301) 2024-12-06T06:13:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465637741, max timestamp=1733465637747Num uniques 859; 2024-12-06T06:13:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465637741, max timestamp=1733465637747Num uniques 859; 2024-12-06T06:13:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465637741, max timestamp=1733465637747Num uniques 859; 2024-12-06T06:13:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:57,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465697774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465697775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465697781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465697783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465697783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T06:13:57,836 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:13:57,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:13:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-06T06:13:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T06:13:57,841 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:13:57,843 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:13:57,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:13:57,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465697893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465697893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465697893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465697894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465697894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T06:13:57,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T06:13:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:57,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8fc31f5c48e64eb898ced3b9f9a765ba 2024-12-06T06:13:58,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465698104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/3eaf5b6e82134a3295147758036055e9 is 50, key is test_row_0/B:col10/1733465637671/Put/seqid=0 2024-12-06T06:13:58,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465698104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465698105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465698105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465698105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741912_1088 (size=12301) 2024-12-06T06:13:58,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T06:13:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T06:13:58,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T06:13:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:58,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465698413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465698414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465698414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465698414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465698414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T06:13:58,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:58,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:13:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T06:13:58,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/3eaf5b6e82134a3295147758036055e9 2024-12-06T06:13:58,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/cf0aff44529b44b59449522de3ecfb27 is 50, key is test_row_0/C:col10/1733465637671/Put/seqid=0 2024-12-06T06:13:58,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741913_1089 (size=12301) 2024-12-06T06:13:58,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/cf0aff44529b44b59449522de3ecfb27 2024-12-06T06:13:58,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8fc31f5c48e64eb898ced3b9f9a765ba as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8fc31f5c48e64eb898ced3b9f9a765ba 2024-12-06T06:13:58,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8fc31f5c48e64eb898ced3b9f9a765ba, entries=150, sequenceid=399, filesize=12.0 K 2024-12-06T06:13:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/3eaf5b6e82134a3295147758036055e9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3eaf5b6e82134a3295147758036055e9 2024-12-06T06:13:58,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3eaf5b6e82134a3295147758036055e9, entries=150, sequenceid=399, filesize=12.0 K 2024-12-06T06:13:58,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/cf0aff44529b44b59449522de3ecfb27 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cf0aff44529b44b59449522de3ecfb27 2024-12-06T06:13:58,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cf0aff44529b44b59449522de3ecfb27, entries=150, sequenceid=399, filesize=12.0 K 2024-12-06T06:13:58,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3860e7c2497c4799e160cda168e0086d in 885ms, sequenceid=399, compaction requested=true 2024-12-06T06:13:58,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:58,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:13:58,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:58,563 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:13:58,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:58,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:58,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:58,563 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:13:58,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:58,565 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:13:58,565 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:58,565 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,565 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5db8ea3d7d2144b69430a7ad24ae22a1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/02e9ec369f6d43728c2032fb374579b1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f3113c5ea43448c1ad35c04ba9117924, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8fc31f5c48e64eb898ced3b9f9a765ba] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=51.2 K 2024-12-06T06:13:58,565 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5db8ea3d7d2144b69430a7ad24ae22a1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733465635469 2024-12-06T06:13:58,565 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:13:58,565 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:13:58,565 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,566 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/80169a8dbb5a4b18a2af081e6a2577e0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/8db2141e77d34ec4ad6198aa68c86404, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a1756085530745d5b7076dde374cb9cc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3eaf5b6e82134a3295147758036055e9] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=48.8 K 2024-12-06T06:13:58,566 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02e9ec369f6d43728c2032fb374579b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733465636123 2024-12-06T06:13:58,567 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 80169a8dbb5a4b18a2af081e6a2577e0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733465635469 2024-12-06T06:13:58,567 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3113c5ea43448c1ad35c04ba9117924, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465636551 2024-12-06T06:13:58,568 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db2141e77d34ec4ad6198aa68c86404, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733465636445 2024-12-06T06:13:58,568 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc31f5c48e64eb898ced3b9f9a765ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465637665 2024-12-06T06:13:58,568 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a1756085530745d5b7076dde374cb9cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465636551 2024-12-06T06:13:58,569 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eaf5b6e82134a3295147758036055e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465637665 2024-12-06T06:13:58,583 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:58,584 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/6d846a1692394375b6b0372b194c077f is 50, key is test_row_0/B:col10/1733465637671/Put/seqid=0 2024-12-06T06:13:58,586 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#77 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:58,587 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/a82df4a5ad6a47ac9490c13848cf3b92 is 50, key is test_row_0/A:col10/1733465637671/Put/seqid=0 2024-12-06T06:13:58,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741914_1090 (size=13153) 2024-12-06T06:13:58,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741915_1091 (size=13187) 2024-12-06T06:13:58,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T06:13:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:58,613 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-06T06:13:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:58,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/b455f7adde9243138f74c2b517c8b3ef is 50, key is test_row_0/A:col10/1733465637781/Put/seqid=0 2024-12-06T06:13:58,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741916_1092 (size=12301) 2024-12-06T06:13:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:58,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:13:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465698933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465698934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465698934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465698936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465698939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T06:13:59,012 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/6d846a1692394375b6b0372b194c077f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6d846a1692394375b6b0372b194c077f 2024-12-06T06:13:59,012 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/a82df4a5ad6a47ac9490c13848cf3b92 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a82df4a5ad6a47ac9490c13848cf3b92 2024-12-06T06:13:59,019 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 6d846a1692394375b6b0372b194c077f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:59,019 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:59,019 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=12, startTime=1733465638563; duration=0sec 2024-12-06T06:13:59,019 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:59,019 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:59,020 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:13:59,020 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into a82df4a5ad6a47ac9490c13848cf3b92(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:59,020 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:59,020 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=12, startTime=1733465638563; duration=0sec 2024-12-06T06:13:59,020 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:59,020 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:59,021 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:13:59,021 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:59,022 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:59,022 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9da458e7fcca4033a79e8bedca558948, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/846c3019add040b9bdf0d0380ecab78b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/66a76128591d4ae6bc327de0e1c209fb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cf0aff44529b44b59449522de3ecfb27] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=48.8 K 2024-12-06T06:13:59,022 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da458e7fcca4033a79e8bedca558948, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733465635469 2024-12-06T06:13:59,023 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 846c3019add040b9bdf0d0380ecab78b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733465636445 2024-12-06T06:13:59,023 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a76128591d4ae6bc327de0e1c209fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465636551 2024-12-06T06:13:59,024 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0aff44529b44b59449522de3ecfb27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465637665 2024-12-06T06:13:59,024 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/b455f7adde9243138f74c2b517c8b3ef 2024-12-06T06:13:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/e6f9541e0791457a883bf326c079907a is 50, key is test_row_0/B:col10/1733465637781/Put/seqid=0 2024-12-06T06:13:59,039 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:59,039 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/18b6680992b8456ca9702124f9e23e6a is 50, key is test_row_0/C:col10/1733465637671/Put/seqid=0 2024-12-06T06:13:59,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741917_1093 (size=12301) 2024-12-06T06:13:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465699044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,046 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/e6f9541e0791457a883bf326c079907a 2024-12-06T06:13:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465699044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741918_1094 (size=13187) 2024-12-06T06:13:59,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465699053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465699054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465699054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1fcc2bac0d9b428f8334d498826eade4 is 50, key is test_row_0/C:col10/1733465637781/Put/seqid=0 2024-12-06T06:13:59,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741919_1095 (size=12301) 2024-12-06T06:13:59,092 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1fcc2bac0d9b428f8334d498826eade4 2024-12-06T06:13:59,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/b455f7adde9243138f74c2b517c8b3ef as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/b455f7adde9243138f74c2b517c8b3ef 2024-12-06T06:13:59,110 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/b455f7adde9243138f74c2b517c8b3ef, entries=150, sequenceid=435, filesize=12.0 K 2024-12-06T06:13:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/e6f9541e0791457a883bf326c079907a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e6f9541e0791457a883bf326c079907a 2024-12-06T06:13:59,122 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e6f9541e0791457a883bf326c079907a, entries=150, sequenceid=435, filesize=12.0 K 2024-12-06T06:13:59,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/1fcc2bac0d9b428f8334d498826eade4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1fcc2bac0d9b428f8334d498826eade4 2024-12-06T06:13:59,130 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1fcc2bac0d9b428f8334d498826eade4, entries=150, sequenceid=435, filesize=12.0 K 2024-12-06T06:13:59,132 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 3860e7c2497c4799e160cda168e0086d in 519ms, sequenceid=435, compaction requested=false 2024-12-06T06:13:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-06T06:13:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-06T06:13:59,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-06T06:13:59,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2910 sec 2024-12-06T06:13:59,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3010 sec 2024-12-06T06:13:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:59,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T06:13:59,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:59,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:59,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:59,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:59,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:59,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:13:59,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8aceaf431e00415cb2db730a1ed882f2 is 50, key is test_row_0/A:col10/1733465638937/Put/seqid=0 2024-12-06T06:13:59,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741920_1096 (size=14741) 2024-12-06T06:13:59,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8aceaf431e00415cb2db730a1ed882f2 2024-12-06T06:13:59,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/6ef6647b82544e81817153c32eb3d9f5 is 50, key is test_row_0/B:col10/1733465638937/Put/seqid=0 2024-12-06T06:13:59,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741921_1097 (size=12301) 2024-12-06T06:13:59,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/6ef6647b82544e81817153c32eb3d9f5 2024-12-06T06:13:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465699293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465699293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465639269, max timestamp=1733465639300Num uniques 859; 2024-12-06T06:13:59,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465639269, max timestamp=1733465639300Num uniques 859; 2024-12-06T06:13:59,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465639269, max timestamp=1733465639300Num uniques 859; 2024-12-06T06:13:59,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:13:59,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/85ed08afb467417e946053f7afe08086 is 50, key is test_row_0/C:col10/1733465638937/Put/seqid=0 2024-12-06T06:13:59,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741922_1098 (size=12301) 2024-12-06T06:13:59,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/85ed08afb467417e946053f7afe08086 2024-12-06T06:13:59,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/8aceaf431e00415cb2db730a1ed882f2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8aceaf431e00415cb2db730a1ed882f2 2024-12-06T06:13:59,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8aceaf431e00415cb2db730a1ed882f2, entries=200, sequenceid=448, filesize=14.4 K 2024-12-06T06:13:59,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/6ef6647b82544e81817153c32eb3d9f5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6ef6647b82544e81817153c32eb3d9f5 2024-12-06T06:13:59,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6ef6647b82544e81817153c32eb3d9f5, entries=150, sequenceid=448, filesize=12.0 K 2024-12-06T06:13:59,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/85ed08afb467417e946053f7afe08086 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/85ed08afb467417e946053f7afe08086 2024-12-06T06:13:59,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/85ed08afb467417e946053f7afe08086, entries=150, sequenceid=448, filesize=12.0 K 2024-12-06T06:13:59,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 3860e7c2497c4799e160cda168e0086d in 105ms, sequenceid=448, compaction requested=true 2024-12-06T06:13:59,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:59,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:13:59,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:59,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:59,362 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:59,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:59,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:13:59,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:13:59,363 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:59,364 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:13:59,364 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:59,364 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a82df4a5ad6a47ac9490c13848cf3b92, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/b455f7adde9243138f74c2b517c8b3ef, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8aceaf431e00415cb2db730a1ed882f2] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=39.3 K 2024-12-06T06:13:59,365 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a82df4a5ad6a47ac9490c13848cf3b92, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465637665 2024-12-06T06:13:59,365 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b455f7adde9243138f74c2b517c8b3ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733465637779 2024-12-06T06:13:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:13:59,366 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aceaf431e00415cb2db730a1ed882f2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733465638932 2024-12-06T06:13:59,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=140.89 KB heapSize=400.69 KB 2024-12-06T06:13:59,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733465639319, max timestamp=1733465639364Num uniques 1068; 2024-12-06T06:13:59,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-12-06T06:13:59,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733465639319, max timestamp=1733465639366Num uniques 1068; 2024-12-06T06:13:59,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-12-06T06:13:59,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733465639319, max timestamp=1733465639364Num uniques 1068; 2024-12-06T06:13:59,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T06:13:59,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:13:59,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:59,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:13:59,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:59,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:13:59,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:13:59,382 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:59,383 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/ff7f2cee18f04278922f43515df84172 is 50, key is test_row_0/A:col10/1733465638937/Put/seqid=0 2024-12-06T06:13:59,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/7a73aaa34a2a4f89b4bf351d691a3450 is 50, key is test_row_0/A:col10/1733465639366/Put/seqid=0 2024-12-06T06:13:59,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741924_1100 (size=17181) 2024-12-06T06:13:59,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/7a73aaa34a2a4f89b4bf351d691a3450 2024-12-06T06:13:59,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741923_1099 (size=13289) 2024-12-06T06:13:59,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/bbeebd448f694c2a8ab4ffc244ef8061 is 50, key is test_row_0/B:col10/1733465639366/Put/seqid=0 2024-12-06T06:13:59,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733465639405, max timestamp=1733465639431Num uniques 859; 2024-12-06T06:13:59,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733465639405, max timestamp=1733465639431Num uniques 859; 2024-12-06T06:13:59,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T06:13:59,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T06:13:59,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733465639405, max timestamp=1733465639431Num uniques 859; 2024-12-06T06:13:59,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T06:13:59,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741925_1101 (size=12301) 2024-12-06T06:13:59,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465699462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465699460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465699464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,472 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/18b6680992b8456ca9702124f9e23e6a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/18b6680992b8456ca9702124f9e23e6a 2024-12-06T06:13:59,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465699466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465699466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,480 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into 18b6680992b8456ca9702124f9e23e6a(size=12.9 K), total size for store is 36.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:59,480 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:59,480 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=12, startTime=1733465638563; duration=0sec 2024-12-06T06:13:59,480 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:13:59,481 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:59,481 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:59,481 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:59,482 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:59,482 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:13:59,482 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:59,482 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6d846a1692394375b6b0372b194c077f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e6f9541e0791457a883bf326c079907a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6ef6647b82544e81817153c32eb3d9f5] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=36.9 K 2024-12-06T06:13:59,483 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d846a1692394375b6b0372b194c077f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465637665 2024-12-06T06:13:59,483 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f9541e0791457a883bf326c079907a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733465637779 2024-12-06T06:13:59,484 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef6647b82544e81817153c32eb3d9f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733465638934 2024-12-06T06:13:59,498 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:59,499 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/138e4531347947699b1a09092a39340d is 50, key is test_row_0/B:col10/1733465638937/Put/seqid=0 2024-12-06T06:13:59,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741926_1102 (size=13255) 2024-12-06T06:13:59,530 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/138e4531347947699b1a09092a39340d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/138e4531347947699b1a09092a39340d 2024-12-06T06:13:59,541 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 138e4531347947699b1a09092a39340d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:59,541 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:59,541 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=13, startTime=1733465639362; duration=0sec 2024-12-06T06:13:59,541 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:13:59,541 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:13:59,541 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:13:59,544 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:13:59,544 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:13:59,544 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:13:59,545 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/18b6680992b8456ca9702124f9e23e6a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1fcc2bac0d9b428f8334d498826eade4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/85ed08afb467417e946053f7afe08086] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=36.9 K 2024-12-06T06:13:59,545 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b6680992b8456ca9702124f9e23e6a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465637665 2024-12-06T06:13:59,546 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fcc2bac0d9b428f8334d498826eade4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733465637779 2024-12-06T06:13:59,546 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ed08afb467417e946053f7afe08086, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733465638934 2024-12-06T06:13:59,562 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#90 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:13:59,563 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/42ee6f9fe6d74c08b215ce0a4444e566 is 50, key is test_row_0/C:col10/1733465638937/Put/seqid=0 2024-12-06T06:13:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741927_1103 (size=13289) 2024-12-06T06:13:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465699574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465699574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465699575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,584 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/42ee6f9fe6d74c08b215ce0a4444e566 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/42ee6f9fe6d74c08b215ce0a4444e566 2024-12-06T06:13:59,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465699584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465699584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,594 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into 42ee6f9fe6d74c08b215ce0a4444e566(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:59,594 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:59,594 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=13, startTime=1733465639362; duration=0sec 2024-12-06T06:13:59,594 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:59,595 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:13:59,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465699783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465699783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465699783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465699794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:13:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465699804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:13:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:13:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:13:59,815 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/ff7f2cee18f04278922f43515df84172 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ff7f2cee18f04278922f43515df84172 2024-12-06T06:13:59,823 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into ff7f2cee18f04278922f43515df84172(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:13:59,823 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:13:59,823 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=13, startTime=1733465639362; duration=0sec 2024-12-06T06:13:59,823 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:13:59,823 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:13:59,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/bbeebd448f694c2a8ab4ffc244ef8061 2024-12-06T06:13:59,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/dc81612b56f54713b02704bb50e2b21a is 50, key is test_row_0/C:col10/1733465639366/Put/seqid=0 2024-12-06T06:13:59,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741928_1104 (size=12301) 2024-12-06T06:13:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T06:13:59,984 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:13:59,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:13:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-06T06:13:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T06:13:59,987 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:13:59,988 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:13:59,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T06:14:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465700093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465700093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465700104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465700104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465700113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T06:14:00,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:00,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:00,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:00,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:00,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:00,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/dc81612b56f54713b02704bb50e2b21a 2024-12-06T06:14:00,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/7a73aaa34a2a4f89b4bf351d691a3450 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7a73aaa34a2a4f89b4bf351d691a3450 2024-12-06T06:14:00,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7a73aaa34a2a4f89b4bf351d691a3450, entries=250, sequenceid=487, filesize=16.8 K 2024-12-06T06:14:00,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/bbeebd448f694c2a8ab4ffc244ef8061 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/bbeebd448f694c2a8ab4ffc244ef8061 2024-12-06T06:14:00,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/bbeebd448f694c2a8ab4ffc244ef8061, entries=150, sequenceid=487, filesize=12.0 K 2024-12-06T06:14:00,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/dc81612b56f54713b02704bb50e2b21a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dc81612b56f54713b02704bb50e2b21a 2024-12-06T06:14:00,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T06:14:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:00,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:00,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T06:14:00,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dc81612b56f54713b02704bb50e2b21a, entries=150, sequenceid=487, filesize=12.0 K 2024-12-06T06:14:00,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.44 KB/52670, heapSize ~125.89 KB/128912, currentSize=147.60 KB/151140 for 3860e7c2497c4799e160cda168e0086d in 940ms, sequenceid=487, compaction requested=false 2024-12-06T06:14:00,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:00,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-06T06:14:00,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:00,452 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=147.60 KB heapSize=401.39 KB 2024-12-06T06:14:00,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:00,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:00,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:00,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:00,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:00,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/22b0411401bd41b184ae32842fcbb3c4 is 50, key is test_row_0/A:col10/1733465639454/Put/seqid=0 2024-12-06T06:14:00,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741929_1105 (size=12301) 2024-12-06T06:14:00,469 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/22b0411401bd41b184ae32842fcbb3c4 2024-12-06T06:14:00,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/79a6bf86facb4b86970d2bd4f14f1c1a is 50, key is test_row_0/B:col10/1733465639454/Put/seqid=0 2024-12-06T06:14:00,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741930_1106 (size=12301) 2024-12-06T06:14:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:00,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T06:14:00,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465700614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465700617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465700618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465700624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465700624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465700724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465700733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:00,888 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/79a6bf86facb4b86970d2bd4f14f1c1a 2024-12-06T06:14:00,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b79eb3c93bb04f31922e3848ff446eb2 is 50, key is test_row_0/C:col10/1733465639454/Put/seqid=0 2024-12-06T06:14:00,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741931_1107 (size=12301) 2024-12-06T06:14:00,902 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b79eb3c93bb04f31922e3848ff446eb2 2024-12-06T06:14:00,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/22b0411401bd41b184ae32842fcbb3c4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/22b0411401bd41b184ae32842fcbb3c4 2024-12-06T06:14:00,914 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/22b0411401bd41b184ae32842fcbb3c4, entries=150, sequenceid=525, filesize=12.0 K 2024-12-06T06:14:00,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-06T06:14:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/79a6bf86facb4b86970d2bd4f14f1c1a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/79a6bf86facb4b86970d2bd4f14f1c1a 2024-12-06T06:14:00,920 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/79a6bf86facb4b86970d2bd4f14f1c1a, entries=150, sequenceid=525, filesize=12.0 K 2024-12-06T06:14:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b79eb3c93bb04f31922e3848ff446eb2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b79eb3c93bb04f31922e3848ff446eb2 2024-12-06T06:14:00,927 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b79eb3c93bb04f31922e3848ff446eb2, entries=150, sequenceid=525, filesize=12.0 K 2024-12-06T06:14:00,929 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~401.34 KB/410976, currentSize=40.25 KB/41220 for 3860e7c2497c4799e160cda168e0086d in 477ms, sequenceid=525, compaction requested=true 2024-12-06T06:14:00,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:00,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:00,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-06T06:14:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-06T06:14:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:00,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:00,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:00,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:00,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:00,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-06T06:14:00,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 946 msec 2024-12-06T06:14:00,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 953 msec 2024-12-06T06:14:00,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/652c5d91c5ad49e394c58619eb99ddae is 50, key is test_row_0/A:col10/1733465640616/Put/seqid=0 2024-12-06T06:14:00,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741932_1108 (size=12301) 2024-12-06T06:14:00,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/652c5d91c5ad49e394c58619eb99ddae 2024-12-06T06:14:00,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/e0e67472579d46c2aeb487a507484540 is 50, key is test_row_0/B:col10/1733465640616/Put/seqid=0 2024-12-06T06:14:00,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741933_1109 (size=12301) 2024-12-06T06:14:00,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/e0e67472579d46c2aeb487a507484540 2024-12-06T06:14:01,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/333efedad9694b748d16167d3c162150 is 50, key is test_row_0/C:col10/1733465640616/Put/seqid=0 2024-12-06T06:14:01,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465640990, max timestamp=1733465641015Num uniques 859; 2024-12-06T06:14:01,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465640990, max timestamp=1733465641015Num uniques 859; 2024-12-06T06:14:01,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:14:01,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:14:01,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465640990, max timestamp=1733465641015Num uniques 859; 2024-12-06T06:14:01,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:14:01,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741934_1110 (size=12301) 2024-12-06T06:14:01,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465701074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465701080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T06:14:01,125 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:01,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-06T06:14:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T06:14:01,129 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:01,130 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:01,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:01,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465701194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465701194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T06:14:01,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T06:14:01,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:01,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:01,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:01,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:01,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:01,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465701404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465701404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/333efedad9694b748d16167d3c162150 2024-12-06T06:14:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/652c5d91c5ad49e394c58619eb99ddae as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/652c5d91c5ad49e394c58619eb99ddae 2024-12-06T06:14:01,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T06:14:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:01,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:01,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T06:14:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:01,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/652c5d91c5ad49e394c58619eb99ddae, entries=150, sequenceid=535, filesize=12.0 K 2024-12-06T06:14:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/e0e67472579d46c2aeb487a507484540 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e0e67472579d46c2aeb487a507484540 2024-12-06T06:14:01,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e0e67472579d46c2aeb487a507484540, entries=150, sequenceid=535, filesize=12.0 K 2024-12-06T06:14:01,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/333efedad9694b748d16167d3c162150 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/333efedad9694b748d16167d3c162150 2024-12-06T06:14:01,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/333efedad9694b748d16167d3c162150, entries=150, sequenceid=535, filesize=12.0 K 2024-12-06T06:14:01,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3860e7c2497c4799e160cda168e0086d in 533ms, sequenceid=535, compaction requested=true 2024-12-06T06:14:01,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:01,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:01,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:01,469 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:01,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:01,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:14:01,470 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:01,472 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:01,472 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:14:01,472 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:01,472 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ff7f2cee18f04278922f43515df84172, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7a73aaa34a2a4f89b4bf351d691a3450, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/22b0411401bd41b184ae32842fcbb3c4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/652c5d91c5ad49e394c58619eb99ddae] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=53.8 K 2024-12-06T06:14:01,473 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7f2cee18f04278922f43515df84172, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733465638934 2024-12-06T06:14:01,473 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:01,473 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:14:01,473 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:01,474 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/138e4531347947699b1a09092a39340d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/bbeebd448f694c2a8ab4ffc244ef8061, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/79a6bf86facb4b86970d2bd4f14f1c1a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e0e67472579d46c2aeb487a507484540] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=49.0 K 2024-12-06T06:14:01,475 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a73aaa34a2a4f89b4bf351d691a3450, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733465639319 2024-12-06T06:14:01,476 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 138e4531347947699b1a09092a39340d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733465638934 2024-12-06T06:14:01,476 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b0411401bd41b184ae32842fcbb3c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733465639454 2024-12-06T06:14:01,477 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbeebd448f694c2a8ab4ffc244ef8061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733465639319 2024-12-06T06:14:01,477 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 652c5d91c5ad49e394c58619eb99ddae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733465640607 2024-12-06T06:14:01,477 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79a6bf86facb4b86970d2bd4f14f1c1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733465639454 2024-12-06T06:14:01,478 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e67472579d46c2aeb487a507484540, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733465640607 2024-12-06T06:14:01,495 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:01,496 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/bb09e525c81046fe9d7a307906a4255a is 50, key is test_row_0/A:col10/1733465640616/Put/seqid=0 2024-12-06T06:14:01,511 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:01,512 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/6033bf1255e94bc49f2ac46fd7cb3463 is 50, key is test_row_0/B:col10/1733465640616/Put/seqid=0 2024-12-06T06:14:01,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741936_1112 (size=13391) 2024-12-06T06:14:01,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741935_1111 (size=13425) 2024-12-06T06:14:01,544 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/bb09e525c81046fe9d7a307906a4255a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bb09e525c81046fe9d7a307906a4255a 2024-12-06T06:14:01,544 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/6033bf1255e94bc49f2ac46fd7cb3463 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6033bf1255e94bc49f2ac46fd7cb3463 2024-12-06T06:14:01,556 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into bb09e525c81046fe9d7a307906a4255a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:01,556 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:01,556 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=12, startTime=1733465641469; duration=0sec 2024-12-06T06:14:01,556 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:01,556 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:14:01,556 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:01,557 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 6033bf1255e94bc49f2ac46fd7cb3463(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:01,557 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:01,557 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=12, startTime=1733465641469; duration=0sec 2024-12-06T06:14:01,557 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:01,558 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:14:01,559 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:01,559 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:14:01,559 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:01,559 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/42ee6f9fe6d74c08b215ce0a4444e566, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dc81612b56f54713b02704bb50e2b21a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b79eb3c93bb04f31922e3848ff446eb2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/333efedad9694b748d16167d3c162150] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=49.0 K 2024-12-06T06:14:01,561 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 42ee6f9fe6d74c08b215ce0a4444e566, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733465638934 2024-12-06T06:14:01,561 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting dc81612b56f54713b02704bb50e2b21a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733465639319 2024-12-06T06:14:01,562 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b79eb3c93bb04f31922e3848ff446eb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733465639454 2024-12-06T06:14:01,562 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 333efedad9694b748d16167d3c162150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733465640607 2024-12-06T06:14:01,583 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:01,584 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/6d180e697bac48d5a869f85f0956e6d7 is 50, key is test_row_0/C:col10/1733465640616/Put/seqid=0 2024-12-06T06:14:01,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741937_1113 (size=13425) 2024-12-06T06:14:01,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-06T06:14:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:01,597 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-06T06:14:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:01,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d344c967927b4120bd22a72fa51d262a is 50, key is test_row_0/A:col10/1733465641051/Put/seqid=0 2024-12-06T06:14:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:01,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:01,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741938_1114 (size=12301) 2024-12-06T06:14:01,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465701645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465701646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465701650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465701723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465701724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T06:14:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465701755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465701754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465701765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465701974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465701974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465701974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,000 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/6d180e697bac48d5a869f85f0956e6d7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/6d180e697bac48d5a869f85f0956e6d7 2024-12-06T06:14:02,008 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into 6d180e697bac48d5a869f85f0956e6d7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:02,008 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:02,008 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=12, startTime=1733465641469; duration=0sec 2024-12-06T06:14:02,009 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:02,009 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:14:02,037 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d344c967927b4120bd22a72fa51d262a 2024-12-06T06:14:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b0a0e3a6ef3344cea32ea33c93674d9a is 50, key is test_row_0/B:col10/1733465641051/Put/seqid=0 2024-12-06T06:14:02,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741939_1115 (size=12301) 2024-12-06T06:14:02,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465702243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465702253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T06:14:02,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465702284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:02,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465702284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465702293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,455 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b0a0e3a6ef3344cea32ea33c93674d9a 2024-12-06T06:14:02,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/60ed9ef0aed94baa8c758b7a4f535ee6 is 50, key is test_row_0/C:col10/1733465641051/Put/seqid=0 2024-12-06T06:14:02,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741940_1116 (size=12301) 2024-12-06T06:14:02,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465702794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:02,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465702794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465702804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:02,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/60ed9ef0aed94baa8c758b7a4f535ee6 2024-12-06T06:14:02,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d344c967927b4120bd22a72fa51d262a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d344c967927b4120bd22a72fa51d262a 2024-12-06T06:14:02,906 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d344c967927b4120bd22a72fa51d262a, entries=150, sequenceid=574, filesize=12.0 K 2024-12-06T06:14:02,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b0a0e3a6ef3344cea32ea33c93674d9a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b0a0e3a6ef3344cea32ea33c93674d9a 2024-12-06T06:14:02,914 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b0a0e3a6ef3344cea32ea33c93674d9a, entries=150, sequenceid=574, filesize=12.0 K 2024-12-06T06:14:02,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/60ed9ef0aed94baa8c758b7a4f535ee6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/60ed9ef0aed94baa8c758b7a4f535ee6 2024-12-06T06:14:02,922 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/60ed9ef0aed94baa8c758b7a4f535ee6, entries=150, sequenceid=574, filesize=12.0 K 2024-12-06T06:14:02,924 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3860e7c2497c4799e160cda168e0086d in 1327ms, sequenceid=574, compaction requested=false 2024-12-06T06:14:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-06T06:14:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-06T06:14:02,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-06T06:14:02,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7960 sec 2024-12-06T06:14:02,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8060 sec 2024-12-06T06:14:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:03,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T06:14:03,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:03,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:03,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:03,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:03,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:03,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:03,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/77ff2dcda37b44d8bd3a6f9db2a2883c is 50, key is test_row_0/A:col10/1733465641625/Put/seqid=0 2024-12-06T06:14:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-06T06:14:03,276 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:03,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-06T06:14:03,280 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T06:14:03,282 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:03,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:03,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741941_1117 (size=14741) 2024-12-06T06:14:03,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/77ff2dcda37b44d8bd3a6f9db2a2883c 2024-12-06T06:14:03,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/66529da7857c4290becdf71d90a227a8 is 50, key is test_row_0/B:col10/1733465641625/Put/seqid=0 2024-12-06T06:14:03,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741942_1118 (size=12301) 2024-12-06T06:14:03,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/66529da7857c4290becdf71d90a227a8 2024-12-06T06:14:03,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/921904a4ea924759a9b6b9cb775d9b72 is 50, key is test_row_0/C:col10/1733465641625/Put/seqid=0 2024-12-06T06:14:03,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741943_1119 (size=12301) 2024-12-06T06:14:03,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465703322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465703324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T06:14:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465703434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465703434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:03,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T06:14:03,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465703644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465703645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/921904a4ea924759a9b6b9cb775d9b72 2024-12-06T06:14:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/77ff2dcda37b44d8bd3a6f9db2a2883c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/77ff2dcda37b44d8bd3a6f9db2a2883c 2024-12-06T06:14:03,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/77ff2dcda37b44d8bd3a6f9db2a2883c, entries=200, sequenceid=587, filesize=14.4 K 2024-12-06T06:14:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/66529da7857c4290becdf71d90a227a8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/66529da7857c4290becdf71d90a227a8 2024-12-06T06:14:03,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:03,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/66529da7857c4290becdf71d90a227a8, entries=150, sequenceid=587, filesize=12.0 K 2024-12-06T06:14:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/921904a4ea924759a9b6b9cb775d9b72 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/921904a4ea924759a9b6b9cb775d9b72 2024-12-06T06:14:03,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/921904a4ea924759a9b6b9cb775d9b72, entries=150, sequenceid=587, filesize=12.0 K 2024-12-06T06:14:03,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3860e7c2497c4799e160cda168e0086d in 488ms, sequenceid=587, compaction requested=true 2024-12-06T06:14:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:03,756 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:14:03,756 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:03,758 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:03,758 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:14:03,758 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,758 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6033bf1255e94bc49f2ac46fd7cb3463, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b0a0e3a6ef3344cea32ea33c93674d9a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/66529da7857c4290becdf71d90a227a8] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=37.1 K 2024-12-06T06:14:03,758 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:03,758 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:14:03,759 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,759 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bb09e525c81046fe9d7a307906a4255a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d344c967927b4120bd22a72fa51d262a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/77ff2dcda37b44d8bd3a6f9db2a2883c] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=39.5 K 2024-12-06T06:14:03,759 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6033bf1255e94bc49f2ac46fd7cb3463, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733465640607 2024-12-06T06:14:03,760 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb09e525c81046fe9d7a307906a4255a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733465640607 2024-12-06T06:14:03,760 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a0e3a6ef3344cea32ea33c93674d9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733465641051 2024-12-06T06:14:03,760 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d344c967927b4120bd22a72fa51d262a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733465641051 2024-12-06T06:14:03,760 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 66529da7857c4290becdf71d90a227a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733465641625 2024-12-06T06:14:03,761 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ff2dcda37b44d8bd3a6f9db2a2883c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733465641625 2024-12-06T06:14:03,775 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:03,776 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/7ae880adf25942bf9113347b65fe37b7 is 50, key is test_row_0/B:col10/1733465641625/Put/seqid=0 2024-12-06T06:14:03,781 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:03,781 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/726697868f544a73a6fcb76eea75acfc is 50, key is test_row_0/A:col10/1733465641625/Put/seqid=0 2024-12-06T06:14:03,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741944_1120 (size=13493) 2024-12-06T06:14:03,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741945_1121 (size=13527) 2024-12-06T06:14:03,812 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/7ae880adf25942bf9113347b65fe37b7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7ae880adf25942bf9113347b65fe37b7 2024-12-06T06:14:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:03,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-06T06:14:03,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:03,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:03,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:03,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:03,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:03,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:03,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/2fac4b2c7690498d9bb53d27e6bf0e18 is 50, key is test_row_0/A:col10/1733465643321/Put/seqid=0 2024-12-06T06:14:03,832 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 7ae880adf25942bf9113347b65fe37b7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:03,832 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:03,832 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=13, startTime=1733465643756; duration=0sec 2024-12-06T06:14:03,832 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:03,832 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:14:03,832 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:03,833 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:03,834 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:14:03,834 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,834 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/6d180e697bac48d5a869f85f0956e6d7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/60ed9ef0aed94baa8c758b7a4f535ee6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/921904a4ea924759a9b6b9cb775d9b72] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=37.1 K 2024-12-06T06:14:03,834 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d180e697bac48d5a869f85f0956e6d7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1733465640607 2024-12-06T06:14:03,835 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ed9ef0aed94baa8c758b7a4f535ee6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733465641051 2024-12-06T06:14:03,836 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 921904a4ea924759a9b6b9cb775d9b72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733465641625 2024-12-06T06:14:03,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465703830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741946_1122 (size=14741) 2024-12-06T06:14:03,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465703836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465703837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,846 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:03,847 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/e7f70980bcaa4362b11ecf85ecf980e8 is 50, key is test_row_0/C:col10/1733465641625/Put/seqid=0 2024-12-06T06:14:03,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741947_1123 (size=13527) 2024-12-06T06:14:03,866 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/e7f70980bcaa4362b11ecf85ecf980e8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e7f70980bcaa4362b11ecf85ecf980e8 2024-12-06T06:14:03,873 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into e7f70980bcaa4362b11ecf85ecf980e8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:03,873 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:03,873 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=13, startTime=1733465643756; duration=0sec 2024-12-06T06:14:03,873 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:03,874 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:14:03,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:03,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T06:14:03,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465703944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465703944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465703944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465703954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465703954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465704154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:04,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465704154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465704154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,220 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/726697868f544a73a6fcb76eea75acfc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/726697868f544a73a6fcb76eea75acfc 2024-12-06T06:14:04,228 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 726697868f544a73a6fcb76eea75acfc(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:04,228 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:04,228 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=13, startTime=1733465643756; duration=0sec 2024-12-06T06:14:04,228 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:04,228 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:14:04,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/2fac4b2c7690498d9bb53d27e6bf0e18 2024-12-06T06:14:04,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/deb1ca81cb854fa29e97720c72734ddc is 50, key is test_row_0/B:col10/1733465643321/Put/seqid=0 2024-12-06T06:14:04,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741948_1124 (size=12301) 2024-12-06T06:14:04,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T06:14:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465704464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465704464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:04,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465704464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:04,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465704465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465704473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:04,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/deb1ca81cb854fa29e97720c72734ddc 2024-12-06T06:14:04,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:04,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/58f4fb2bcf434d1b8dc1b7dd78cb0447 is 50, key is test_row_0/C:col10/1733465643321/Put/seqid=0 2024-12-06T06:14:04,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741949_1125 (size=12301) 2024-12-06T06:14:04,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/58f4fb2bcf434d1b8dc1b7dd78cb0447 2024-12-06T06:14:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/2fac4b2c7690498d9bb53d27e6bf0e18 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/2fac4b2c7690498d9bb53d27e6bf0e18 2024-12-06T06:14:04,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/2fac4b2c7690498d9bb53d27e6bf0e18, entries=200, sequenceid=608, filesize=14.4 K 2024-12-06T06:14:04,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/deb1ca81cb854fa29e97720c72734ddc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/deb1ca81cb854fa29e97720c72734ddc 2024-12-06T06:14:04,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/deb1ca81cb854fa29e97720c72734ddc, entries=150, sequenceid=608, filesize=12.0 K 2024-12-06T06:14:04,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/58f4fb2bcf434d1b8dc1b7dd78cb0447 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/58f4fb2bcf434d1b8dc1b7dd78cb0447 2024-12-06T06:14:04,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/58f4fb2bcf434d1b8dc1b7dd78cb0447, entries=150, sequenceid=608, filesize=12.0 K 2024-12-06T06:14:04,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 3860e7c2497c4799e160cda168e0086d in 910ms, sequenceid=608, compaction requested=false 2024-12-06T06:14:04,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:04,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-06T06:14:04,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:04,820 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T06:14:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:04,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/dee15741e2c84a178ca9946072475950 is 50, key is test_row_0/A:col10/1733465643822/Put/seqid=0 2024-12-06T06:14:04,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741950_1126 (size=12301) 2024-12-06T06:14:04,855 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/dee15741e2c84a178ca9946072475950 2024-12-06T06:14:04,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/91f8e46501ca4f75af741d25aa04bbdb is 50, key is test_row_0/B:col10/1733465643822/Put/seqid=0 2024-12-06T06:14:04,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741951_1127 (size=12301) 2024-12-06T06:14:04,884 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/91f8e46501ca4f75af741d25aa04bbdb 2024-12-06T06:14:04,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/3f6b616ccf3946259b5eb2f6b74d8aaa is 50, key is test_row_0/C:col10/1733465643822/Put/seqid=0 2024-12-06T06:14:04,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741952_1128 (size=12301) 2024-12-06T06:14:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:04,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:05,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465645011, max timestamp=1733465645020Num uniques 859; 2024-12-06T06:14:05,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465645011, max timestamp=1733465645020Num uniques 859; 2024-12-06T06:14:05,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:05,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465645011, max timestamp=1733465645020Num uniques 859; 2024-12-06T06:14:05,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:05,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465705060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465705061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465705066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465705173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465705184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465705184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,326 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/3f6b616ccf3946259b5eb2f6b74d8aaa 2024-12-06T06:14:05,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/dee15741e2c84a178ca9946072475950 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/dee15741e2c84a178ca9946072475950 2024-12-06T06:14:05,343 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/dee15741e2c84a178ca9946072475950, entries=150, sequenceid=622, filesize=12.0 K 2024-12-06T06:14:05,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/91f8e46501ca4f75af741d25aa04bbdb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/91f8e46501ca4f75af741d25aa04bbdb 2024-12-06T06:14:05,351 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/91f8e46501ca4f75af741d25aa04bbdb, entries=150, sequenceid=622, filesize=12.0 K 2024-12-06T06:14:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/3f6b616ccf3946259b5eb2f6b74d8aaa as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/3f6b616ccf3946259b5eb2f6b74d8aaa 2024-12-06T06:14:05,357 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/3f6b616ccf3946259b5eb2f6b74d8aaa, entries=150, sequenceid=622, filesize=12.0 K 2024-12-06T06:14:05,358 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3860e7c2497c4799e160cda168e0086d in 538ms, sequenceid=622, compaction requested=true 2024-12-06T06:14:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-06T06:14:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-06T06:14:05,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-06T06:14:05,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0780 sec 2024-12-06T06:14:05,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0870 sec 2024-12-06T06:14:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:05,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-06T06:14:05,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:05,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:05,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:05,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:05,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:05,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:05,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d86faf96f98844ea91fe404c5a0edf47 is 50, key is test_row_0/A:col10/1733465645062/Put/seqid=0 2024-12-06T06:14:05,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741953_1129 (size=14741) 2024-12-06T06:14:05,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d86faf96f98844ea91fe404c5a0edf47 2024-12-06T06:14:05,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/eb877eb35a7844cc9b516575fec7405d is 50, key is test_row_0/B:col10/1733465645062/Put/seqid=0 2024-12-06T06:14:05,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741954_1130 (size=12301) 2024-12-06T06:14:05,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465705410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/eb877eb35a7844cc9b516575fec7405d 2024-12-06T06:14:05,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465705415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465705416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T06:14:05,424 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:05,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-06T06:14:05,427 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:05,428 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:05,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-06T06:14:05,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/550b5dc9717e436089057e1701ec261e is 50, key is test_row_0/C:col10/1733465645062/Put/seqid=0 2024-12-06T06:14:05,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741955_1131 (size=12301) 2024-12-06T06:14:05,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/550b5dc9717e436089057e1701ec261e 2024-12-06T06:14:05,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/d86faf96f98844ea91fe404c5a0edf47 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d86faf96f98844ea91fe404c5a0edf47 2024-12-06T06:14:05,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d86faf96f98844ea91fe404c5a0edf47, entries=200, sequenceid=658, filesize=14.4 K 2024-12-06T06:14:05,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/eb877eb35a7844cc9b516575fec7405d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/eb877eb35a7844cc9b516575fec7405d 2024-12-06T06:14:05,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/eb877eb35a7844cc9b516575fec7405d, entries=150, sequenceid=658, filesize=12.0 K 2024-12-06T06:14:05,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/550b5dc9717e436089057e1701ec261e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/550b5dc9717e436089057e1701ec261e 2024-12-06T06:14:05,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465705483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465705484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/550b5dc9717e436089057e1701ec261e, entries=150, sequenceid=658, filesize=12.0 K 2024-12-06T06:14:05,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 3860e7c2497c4799e160cda168e0086d in 107ms, sequenceid=658, compaction requested=true 2024-12-06T06:14:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:05,495 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:05,495 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:05,494 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in 3860e7c2497c4799e160cda168e0086d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,494 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in 3860e7c2497c4799e160cda168e0086d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,498 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55310 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:05,498 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:05,499 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:14:05,499 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:14:05,499 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,499 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,499 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/726697868f544a73a6fcb76eea75acfc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/2fac4b2c7690498d9bb53d27e6bf0e18, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/dee15741e2c84a178ca9946072475950, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d86faf96f98844ea91fe404c5a0edf47] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=54.0 K 2024-12-06T06:14:05,499 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7ae880adf25942bf9113347b65fe37b7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/deb1ca81cb854fa29e97720c72734ddc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/91f8e46501ca4f75af741d25aa04bbdb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/eb877eb35a7844cc9b516575fec7405d] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=49.2 K 2024-12-06T06:14:05,500 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 726697868f544a73a6fcb76eea75acfc, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733465641625 2024-12-06T06:14:05,500 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae880adf25942bf9113347b65fe37b7, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733465641625 2024-12-06T06:14:05,500 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting deb1ca81cb854fa29e97720c72734ddc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733465643320 2024-12-06T06:14:05,500 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fac4b2c7690498d9bb53d27e6bf0e18, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733465643311 2024-12-06T06:14:05,500 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 91f8e46501ca4f75af741d25aa04bbdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733465643820 2024-12-06T06:14:05,500 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dee15741e2c84a178ca9946072475950, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733465643820 2024-12-06T06:14:05,501 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting eb877eb35a7844cc9b516575fec7405d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1733465645057 2024-12-06T06:14:05,501 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d86faf96f98844ea91fe404c5a0edf47, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1733465645055 2024-12-06T06:14:05,513 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:05,513 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/80a65d435100477a8d48ab5f9f628c67 is 50, key is test_row_0/A:col10/1733465645062/Put/seqid=0 2024-12-06T06:14:05,516 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:05,517 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/dcf5da29726d4438b6a5a47d1a3ed71b is 50, key is test_row_0/B:col10/1733465645062/Put/seqid=0 2024-12-06T06:14:05,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741956_1132 (size=13663) 2024-12-06T06:14:05,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741957_1133 (size=13629) 2024-12-06T06:14:05,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T06:14:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-06T06:14:05,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:05,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:05,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:05,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:05,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:05,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:05,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/30ee758e20ea40aea3bed31f6d4e2c72 is 50, key is test_row_0/A:col10/1733465645407/Put/seqid=0 2024-12-06T06:14:05,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741958_1134 (size=14741) 2024-12-06T06:14:05,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733465645549, max timestamp=1733465645571Num uniques 859; 2024-12-06T06:14:05,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733465645549, max timestamp=1733465645571Num uniques 859; 2024-12-06T06:14:05,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733465645549, max timestamp=1733465645571Num uniques 859; 2024-12-06T06:14:05,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T06:14:05,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T06:14:05,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T06:14:05,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-06T06:14:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465705605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465705606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465705609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465705714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465705714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465705714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-06T06:14:05,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:05,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-06T06:14:05,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-06T06:14:05,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:05,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:05,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465705923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465705923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465705924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:05,933 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/80a65d435100477a8d48ab5f9f628c67 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80a65d435100477a8d48ab5f9f628c67 2024-12-06T06:14:05,939 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/dcf5da29726d4438b6a5a47d1a3ed71b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/dcf5da29726d4438b6a5a47d1a3ed71b 2024-12-06T06:14:05,942 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 80a65d435100477a8d48ab5f9f628c67(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:05,942 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:05,942 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=12, startTime=1733465645495; duration=0sec 2024-12-06T06:14:05,942 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:05,942 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:14:05,942 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:05,944 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:05,944 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:14:05,944 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:05,944 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e7f70980bcaa4362b11ecf85ecf980e8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/58f4fb2bcf434d1b8dc1b7dd78cb0447, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/3f6b616ccf3946259b5eb2f6b74d8aaa, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/550b5dc9717e436089057e1701ec261e] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=49.2 K 2024-12-06T06:14:05,946 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f70980bcaa4362b11ecf85ecf980e8, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733465641625 2024-12-06T06:14:05,947 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into dcf5da29726d4438b6a5a47d1a3ed71b(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:05,947 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:05,947 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=12, startTime=1733465645495; duration=0sec 2024-12-06T06:14:05,947 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58f4fb2bcf434d1b8dc1b7dd78cb0447, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733465643320 2024-12-06T06:14:05,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/30ee758e20ea40aea3bed31f6d4e2c72 2024-12-06T06:14:05,950 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f6b616ccf3946259b5eb2f6b74d8aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733465643820 2024-12-06T06:14:05,950 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:05,950 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:14:05,950 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 550b5dc9717e436089057e1701ec261e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1733465645057 2024-12-06T06:14:05,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/461a7ebab2ea42b9a60e487d20d44f21 is 50, key is test_row_0/B:col10/1733465645407/Put/seqid=0 2024-12-06T06:14:05,970 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:05,971 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b43b0411c92941a5973831c10f1433e2 is 50, key is test_row_0/C:col10/1733465645062/Put/seqid=0 2024-12-06T06:14:05,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741960_1136 (size=13663) 2024-12-06T06:14:05,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741959_1135 (size=12301) 2024-12-06T06:14:05,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/461a7ebab2ea42b9a60e487d20d44f21 2024-12-06T06:14:05,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b073d80de0924c6a880b8b802ba3ec1e is 50, key is test_row_0/C:col10/1733465645407/Put/seqid=0 2024-12-06T06:14:06,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741961_1137 (size=12301) 2024-12-06T06:14:06,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b073d80de0924c6a880b8b802ba3ec1e 2024-12-06T06:14:06,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/30ee758e20ea40aea3bed31f6d4e2c72 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/30ee758e20ea40aea3bed31f6d4e2c72 2024-12-06T06:14:06,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/30ee758e20ea40aea3bed31f6d4e2c72, entries=200, sequenceid=669, filesize=14.4 K 2024-12-06T06:14:06,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/461a7ebab2ea42b9a60e487d20d44f21 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/461a7ebab2ea42b9a60e487d20d44f21 2024-12-06T06:14:06,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/461a7ebab2ea42b9a60e487d20d44f21, entries=150, sequenceid=669, filesize=12.0 K 2024-12-06T06:14:06,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b073d80de0924c6a880b8b802ba3ec1e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b073d80de0924c6a880b8b802ba3ec1e 2024-12-06T06:14:06,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b073d80de0924c6a880b8b802ba3ec1e, entries=150, sequenceid=669, filesize=12.0 K 2024-12-06T06:14:06,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 3860e7c2497c4799e160cda168e0086d in 506ms, sequenceid=669, compaction requested=false 2024-12-06T06:14:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:06,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-06T06:14:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:06,046 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-06T06:14:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/80b96c02d272407fab8c93945fdb09b6 is 50, key is test_row_0/A:col10/1733465645608/Put/seqid=0 2024-12-06T06:14:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-06T06:14:06,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741962_1138 (size=12301) 2024-12-06T06:14:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. as already flushing 2024-12-06T06:14:06,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465706246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465706253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465706253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,383 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/b43b0411c92941a5973831c10f1433e2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b43b0411c92941a5973831c10f1433e2 2024-12-06T06:14:06,397 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into b43b0411c92941a5973831c10f1433e2(size=13.3 K), total size for store is 25.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:06,397 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:06,397 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=12, startTime=1733465645495; duration=0sec 2024-12-06T06:14:06,397 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:06,397 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:14:06,458 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/80b96c02d272407fab8c93945fdb09b6 2024-12-06T06:14:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/20003e4475b946cb92d21026399708f9 is 50, key is test_row_0/B:col10/1733465645608/Put/seqid=0 2024-12-06T06:14:06,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741963_1139 (size=12301) 2024-12-06T06:14:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-06T06:14:06,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465706573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465706584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465706584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,877 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/20003e4475b946cb92d21026399708f9 2024-12-06T06:14:06,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465706883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/91752b6deb7b4bcd831be8762be7d981 is 50, key is test_row_0/C:col10/1733465645608/Put/seqid=0 2024-12-06T06:14:06,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741964_1140 (size=12301) 2024-12-06T06:14:06,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465706894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465706894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,293 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/91752b6deb7b4bcd831be8762be7d981 2024-12-06T06:14:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/80b96c02d272407fab8c93945fdb09b6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80b96c02d272407fab8c93945fdb09b6 2024-12-06T06:14:07,307 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80b96c02d272407fab8c93945fdb09b6, entries=150, sequenceid=705, filesize=12.0 K 2024-12-06T06:14:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/20003e4475b946cb92d21026399708f9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/20003e4475b946cb92d21026399708f9 2024-12-06T06:14:07,316 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/20003e4475b946cb92d21026399708f9, entries=150, sequenceid=705, filesize=12.0 K 2024-12-06T06:14:07,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/91752b6deb7b4bcd831be8762be7d981 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/91752b6deb7b4bcd831be8762be7d981 2024-12-06T06:14:07,339 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/91752b6deb7b4bcd831be8762be7d981, entries=150, sequenceid=705, filesize=12.0 K 2024-12-06T06:14:07,347 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=60.38 KB/61830 for 3860e7c2497c4799e160cda168e0086d in 1301ms, sequenceid=705, compaction requested=true 2024-12-06T06:14:07,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-06T06:14:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-06T06:14:07,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-06T06:14:07,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9210 sec 2024-12-06T06:14:07,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.9280 sec 2024-12-06T06:14:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:07,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:14:07,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:07,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:07,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:07,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:07,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:07,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:07,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4dc3da7853bc47658ab756d6e296ff16 is 50, key is test_row_0/A:col10/1733465647394/Put/seqid=0 2024-12-06T06:14:07,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741965_1141 (size=14741) 2024-12-06T06:14:07,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=719 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4dc3da7853bc47658ab756d6e296ff16 2024-12-06T06:14:07,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b44730a7147e4483b0f0c953d79ce8a1 is 50, key is test_row_0/B:col10/1733465647394/Put/seqid=0 2024-12-06T06:14:07,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741966_1142 (size=12301) 2024-12-06T06:14:07,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465707442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465707442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465707444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,464 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-06T06:14:07,464 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:07,464 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-06T06:14:07,464 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:07,468 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-06T06:14:07,468 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:07,469 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-06T06:14:07,469 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:07,470 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:07,470 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:07,470 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:07,470 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:07,470 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:07,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:07,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:07,470 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:07,470 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:07,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:07,471 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:07,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:07,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38974 deadline: 1733465707514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,515 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in 3860e7c2497c4799e160cda168e0086d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38932 deadline: 1733465707524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,525 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in 3860e7c2497c4799e160cda168e0086d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465707553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465707553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465707553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-06T06:14:07,574 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465707763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465707763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465707763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:07,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=719 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b44730a7147e4483b0f0c953d79ce8a1 2024-12-06T06:14:07,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/dd3d20afd51749d79d2986fcb94392f0 is 50, key is test_row_0/C:col10/1733465647394/Put/seqid=0 2024-12-06T06:14:07,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741967_1143 (size=12301) 2024-12-06T06:14:08,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:08,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:08,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38948 deadline: 1733465708074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38930 deadline: 1733465708074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38928 deadline: 1733465708074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:08,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=719 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/dd3d20afd51749d79d2986fcb94392f0 2024-12-06T06:14:08,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/4dc3da7853bc47658ab756d6e296ff16 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dc3da7853bc47658ab756d6e296ff16 2024-12-06T06:14:08,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dc3da7853bc47658ab756d6e296ff16, entries=200, sequenceid=719, filesize=14.4 K 2024-12-06T06:14:08,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/b44730a7147e4483b0f0c953d79ce8a1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b44730a7147e4483b0f0c953d79ce8a1 2024-12-06T06:14:08,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b44730a7147e4483b0f0c953d79ce8a1, entries=150, sequenceid=719, filesize=12.0 K 2024-12-06T06:14:08,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/dd3d20afd51749d79d2986fcb94392f0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dd3d20afd51749d79d2986fcb94392f0 2024-12-06T06:14:08,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dd3d20afd51749d79d2986fcb94392f0, entries=150, sequenceid=719, filesize=12.0 K 2024-12-06T06:14:08,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3860e7c2497c4799e160cda168e0086d in 893ms, sequenceid=719, compaction requested=true 2024-12-06T06:14:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:08,289 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:08,290 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3860e7c2497c4799e160cda168e0086d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:08,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:08,291 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55446 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:08,291 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/A is initiating minor compaction (all files) 2024-12-06T06:14:08,291 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/A in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:08,291 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80a65d435100477a8d48ab5f9f628c67, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/30ee758e20ea40aea3bed31f6d4e2c72, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80b96c02d272407fab8c93945fdb09b6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dc3da7853bc47658ab756d6e296ff16] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=54.1 K 2024-12-06T06:14:08,292 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80a65d435100477a8d48ab5f9f628c67, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1733465645057 2024-12-06T06:14:08,292 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:08,292 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/B is initiating minor compaction (all files) 2024-12-06T06:14:08,292 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/B in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:08,292 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/dcf5da29726d4438b6a5a47d1a3ed71b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/461a7ebab2ea42b9a60e487d20d44f21, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/20003e4475b946cb92d21026399708f9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b44730a7147e4483b0f0c953d79ce8a1] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=49.3 K 2024-12-06T06:14:08,292 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ee758e20ea40aea3bed31f6d4e2c72, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1733465645407 2024-12-06T06:14:08,293 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf5da29726d4438b6a5a47d1a3ed71b, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1733465645057 2024-12-06T06:14:08,293 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b96c02d272407fab8c93945fdb09b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733465645571 2024-12-06T06:14:08,293 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 461a7ebab2ea42b9a60e487d20d44f21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1733465645407 2024-12-06T06:14:08,293 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 20003e4475b946cb92d21026399708f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733465645571 2024-12-06T06:14:08,294 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc3da7853bc47658ab756d6e296ff16, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=719, earliestPutTs=1733465646243 2024-12-06T06:14:08,294 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b44730a7147e4483b0f0c953d79ce8a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=719, earliestPutTs=1733465646243 2024-12-06T06:14:08,307 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#A#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:08,307 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:08,308 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/227c760ea76f4b288285ffa7077a8331 is 50, key is test_row_0/A:col10/1733465647394/Put/seqid=0 2024-12-06T06:14:08,308 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5b85b6566a6d46468f2a19164910ed3d is 50, key is test_row_0/B:col10/1733465647394/Put/seqid=0 2024-12-06T06:14:08,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741969_1145 (size=13799) 2024-12-06T06:14:08,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741968_1144 (size=13765) 2024-12-06T06:14:08,331 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/227c760ea76f4b288285ffa7077a8331 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/227c760ea76f4b288285ffa7077a8331 2024-12-06T06:14:08,331 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/5b85b6566a6d46468f2a19164910ed3d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5b85b6566a6d46468f2a19164910ed3d 2024-12-06T06:14:08,339 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/A of 3860e7c2497c4799e160cda168e0086d into 227c760ea76f4b288285ffa7077a8331(size=13.5 K), total size for store is 13.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:08,339 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:08,339 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/A, priority=12, startTime=1733465648289; duration=0sec 2024-12-06T06:14:08,339 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:08,339 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:A 2024-12-06T06:14:08,340 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:08,340 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/B of 3860e7c2497c4799e160cda168e0086d into 5b85b6566a6d46468f2a19164910ed3d(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:08,340 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:08,340 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/B, priority=12, startTime=1733465648290; duration=0sec 2024-12-06T06:14:08,340 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:08,340 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:B 2024-12-06T06:14:08,341 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:08,341 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 3860e7c2497c4799e160cda168e0086d/C is initiating minor compaction (all files) 2024-12-06T06:14:08,342 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3860e7c2497c4799e160cda168e0086d/C in TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:08,342 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b43b0411c92941a5973831c10f1433e2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b073d80de0924c6a880b8b802ba3ec1e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/91752b6deb7b4bcd831be8762be7d981, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dd3d20afd51749d79d2986fcb94392f0] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp, totalSize=49.4 K 2024-12-06T06:14:08,342 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b43b0411c92941a5973831c10f1433e2, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1733465645057 2024-12-06T06:14:08,342 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b073d80de0924c6a880b8b802ba3ec1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1733465645407 2024-12-06T06:14:08,343 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91752b6deb7b4bcd831be8762be7d981, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733465645571 2024-12-06T06:14:08,343 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd3d20afd51749d79d2986fcb94392f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=719, earliestPutTs=1733465646243 2024-12-06T06:14:08,352 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3860e7c2497c4799e160cda168e0086d#C#compaction#133 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:08,353 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/90a0bf76258b44c1b669ec592217b6f4 is 50, key is test_row_0/C:col10/1733465647394/Put/seqid=0 2024-12-06T06:14:08,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741970_1146 (size=13799) 2024-12-06T06:14:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:08,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-06T06:14:08,589 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-06T06:14:08,589 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:08,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1733465647418, max timestamp=1733465648585Num uniques 859; 2024-12-06T06:14:08,589 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:08,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:08,589 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:08,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:08,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733465647418, max timestamp=1733465648585Num uniques 859; 2024-12-06T06:14:08,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=7872, suffix heap size=117064,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=172 2024-12-06T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:08,591 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-06T06:14:08,591 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:08,591 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:08,592 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:08,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:08,592 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-06T06:14:08,592 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:08,592 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:08,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:08,592 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:08,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/5a37211820004a5bab24b539e5f087e3 is 50, key is test_row_0/A:col10/1733465647437/Put/seqid=0 2024-12-06T06:14:08,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741971_1147 (size=12301) 2024-12-06T06:14:08,765 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/90a0bf76258b44c1b669ec592217b6f4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90a0bf76258b44c1b669ec592217b6f4 2024-12-06T06:14:08,770 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3860e7c2497c4799e160cda168e0086d/C of 3860e7c2497c4799e160cda168e0086d into 90a0bf76258b44c1b669ec592217b6f4(size=13.5 K), total size for store is 13.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:08,771 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:08,771 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., storeName=3860e7c2497c4799e160cda168e0086d/C, priority=12, startTime=1733465648290; duration=0sec 2024-12-06T06:14:08,771 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:08,771 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3860e7c2497c4799e160cda168e0086d:C 2024-12-06T06:14:09,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/5a37211820004a5bab24b539e5f087e3 2024-12-06T06:14:09,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/50b481e6feac4e42932e0ef1722820d2 is 50, key is test_row_0/B:col10/1733465647437/Put/seqid=0 2024-12-06T06:14:09,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741972_1148 (size=12301) 2024-12-06T06:14:09,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.77 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/50b481e6feac4e42932e0ef1722820d2 2024-12-06T06:14:09,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/9026a0d7abd14d05b3f245c5e14d57f1 is 50, key is test_row_0/C:col10/1733465647437/Put/seqid=0 2024-12-06T06:14:09,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741973_1149 (size=12301) 2024-12-06T06:14:09,290 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T06:14:09,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/9026a0d7abd14d05b3f245c5e14d57f1 2024-12-06T06:14:09,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/5a37211820004a5bab24b539e5f087e3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5a37211820004a5bab24b539e5f087e3 2024-12-06T06:14:09,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5a37211820004a5bab24b539e5f087e3, entries=150, sequenceid=743, filesize=12.0 K 2024-12-06T06:14:09,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/50b481e6feac4e42932e0ef1722820d2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/50b481e6feac4e42932e0ef1722820d2 2024-12-06T06:14:09,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/50b481e6feac4e42932e0ef1722820d2, entries=150, sequenceid=743, filesize=12.0 K 2024-12-06T06:14:09,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/9026a0d7abd14d05b3f245c5e14d57f1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9026a0d7abd14d05b3f245c5e14d57f1 2024-12-06T06:14:09,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9026a0d7abd14d05b3f245c5e14d57f1, entries=150, sequenceid=743, filesize=12.0 K 2024-12-06T06:14:09,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.67 KB/69298, heapSize ~190.45 KB/195016, currentSize=0 B/0 for 3860e7c2497c4799e160cda168e0086d in 885ms, sequenceid=743, compaction requested=false 2024-12-06T06:14:09,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3860e7c2497c4799e160cda168e0086d: 2024-12-06T06:14:11,546 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-06T06:14:11,546 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:11,546 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:11,546 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:11,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:11,554 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-06T06:14:11,555 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:11,555 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:11,555 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:11,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2523 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2356 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1111 2024-12-06T06:14:11,555 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3332 rows 2024-12-06T06:14:11,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-12-06T06:14:11,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3310 rows 2024-12-06T06:14:11,556 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T06:14:11,556 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:14:11,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:11,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:11,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:11,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T06:14:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T06:14:11,567 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465651567"}]},"ts":"1733465651567"} 2024-12-06T06:14:11,569 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T06:14:11,569 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T06:14:11,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:14:11,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, UNASSIGN}] 2024-12-06T06:14:11,575 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, UNASSIGN 2024-12-06T06:14:11,577 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3860e7c2497c4799e160cda168e0086d, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:11,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, UNASSIGN because future has completed 2024-12-06T06:14:11,579 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:14:11,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:14:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T06:14:11,735 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:11,735 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:14:11,736 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 3860e7c2497c4799e160cda168e0086d, disabling compactions & flushes 2024-12-06T06:14:11,736 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:11,736 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:11,736 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. after waiting 0 ms 2024-12-06T06:14:11,736 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:11,736 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 3860e7c2497c4799e160cda168e0086d 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-06T06:14:11,736 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=A 2024-12-06T06:14:11,737 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:11,737 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=B 2024-12-06T06:14:11,737 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:11,737 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3860e7c2497c4799e160cda168e0086d, store=C 2024-12-06T06:14:11,737 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:11,741 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/f71c56d6ab92445aad9fcc896911e766 is 50, key is test_row_1/A:col10/1733465651553/Put/seqid=0 2024-12-06T06:14:11,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741974_1150 (size=9857) 2024-12-06T06:14:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T06:14:12,145 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=749 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/f71c56d6ab92445aad9fcc896911e766 2024-12-06T06:14:12,153 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/9128d85565204a62ab4bc4091cd85a26 is 50, key is test_row_1/B:col10/1733465651553/Put/seqid=0 2024-12-06T06:14:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741975_1151 (size=9857) 2024-12-06T06:14:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T06:14:12,557 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=749 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/9128d85565204a62ab4bc4091cd85a26 2024-12-06T06:14:12,564 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/9213a8d814984abfb206c7fb6cb7a3fa is 50, key is test_row_1/C:col10/1733465651553/Put/seqid=0 2024-12-06T06:14:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741976_1152 (size=9857) 2024-12-06T06:14:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T06:14:12,969 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=749 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/9213a8d814984abfb206c7fb6cb7a3fa 2024-12-06T06:14:12,973 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/A/f71c56d6ab92445aad9fcc896911e766 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f71c56d6ab92445aad9fcc896911e766 2024-12-06T06:14:12,977 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f71c56d6ab92445aad9fcc896911e766, entries=100, sequenceid=749, filesize=9.6 K 2024-12-06T06:14:12,978 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/B/9128d85565204a62ab4bc4091cd85a26 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/9128d85565204a62ab4bc4091cd85a26 2024-12-06T06:14:12,981 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/9128d85565204a62ab4bc4091cd85a26, entries=100, sequenceid=749, filesize=9.6 K 2024-12-06T06:14:12,982 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/.tmp/C/9213a8d814984abfb206c7fb6cb7a3fa as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9213a8d814984abfb206c7fb6cb7a3fa 2024-12-06T06:14:12,986 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9213a8d814984abfb206c7fb6cb7a3fa, entries=100, sequenceid=749, filesize=9.6 K 2024-12-06T06:14:12,987 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 3860e7c2497c4799e160cda168e0086d in 1251ms, sequenceid=749, compaction requested=true 2024-12-06T06:14:12,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/0b456b12b0b6409da957aeaf3ee0a5e0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd81269cb2404307a1331c877dd0dade, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/74727690ca4641d8a26f91279b68b50a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/915c4d8803d940c88ecd64d2b74f0d57, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/26932b711a5344c481cc333feb3d995c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4781812c93c5412088f14929e57bdb48, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8ad1e85278214a2fb0236f038e4072f4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dd7dfac129744b99474900b3e8842b6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d93a98ca1a4e428bb920ddefbc3d3878, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/244ea4819c8e4521b752d1d82de11602, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/192ffe9db3d5465789582046e27ca263, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a68b3fb71520459f9c10beced8846fcb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/6c71a0c3dece4687bdee6f2149235b2c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8a7604e939c641f5afaa4a54677ab410, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7657638d83b94390b15d547bc552d6d9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ea99e8af47da41ed84943f4177029237, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/93d6c6b25c3e491b8cb8afabd769d370, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a9633258a7db49b3b3d21d8230a1b0b1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e3a326a45d7b4b209da9c3810a9efcff, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd946af21f714263a77e8df7c2176a05, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5db8ea3d7d2144b69430a7ad24ae22a1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e53aeabf26e94efeb725a66f3ad5514f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/02e9ec369f6d43728c2032fb374579b1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f3113c5ea43448c1ad35c04ba9117924, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a82df4a5ad6a47ac9490c13848cf3b92, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8fc31f5c48e64eb898ced3b9f9a765ba, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/b455f7adde9243138f74c2b517c8b3ef, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8aceaf431e00415cb2db730a1ed882f2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ff7f2cee18f04278922f43515df84172, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7a73aaa34a2a4f89b4bf351d691a3450, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/22b0411401bd41b184ae32842fcbb3c4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bb09e525c81046fe9d7a307906a4255a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/652c5d91c5ad49e394c58619eb99ddae, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d344c967927b4120bd22a72fa51d262a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/77ff2dcda37b44d8bd3a6f9db2a2883c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/726697868f544a73a6fcb76eea75acfc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/2fac4b2c7690498d9bb53d27e6bf0e18, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/dee15741e2c84a178ca9946072475950, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d86faf96f98844ea91fe404c5a0edf47, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80a65d435100477a8d48ab5f9f628c67, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/30ee758e20ea40aea3bed31f6d4e2c72, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80b96c02d272407fab8c93945fdb09b6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dc3da7853bc47658ab756d6e296ff16] to archive 2024-12-06T06:14:12,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:14:12,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/0b456b12b0b6409da957aeaf3ee0a5e0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/0b456b12b0b6409da957aeaf3ee0a5e0 2024-12-06T06:14:12,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd81269cb2404307a1331c877dd0dade to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd81269cb2404307a1331c877dd0dade 2024-12-06T06:14:12,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/74727690ca4641d8a26f91279b68b50a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/74727690ca4641d8a26f91279b68b50a 2024-12-06T06:14:12,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/915c4d8803d940c88ecd64d2b74f0d57 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/915c4d8803d940c88ecd64d2b74f0d57 2024-12-06T06:14:13,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/26932b711a5344c481cc333feb3d995c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/26932b711a5344c481cc333feb3d995c 2024-12-06T06:14:13,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4781812c93c5412088f14929e57bdb48 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4781812c93c5412088f14929e57bdb48 2024-12-06T06:14:13,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8ad1e85278214a2fb0236f038e4072f4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8ad1e85278214a2fb0236f038e4072f4 2024-12-06T06:14:13,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dd7dfac129744b99474900b3e8842b6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dd7dfac129744b99474900b3e8842b6 2024-12-06T06:14:13,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d93a98ca1a4e428bb920ddefbc3d3878 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d93a98ca1a4e428bb920ddefbc3d3878 2024-12-06T06:14:13,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/244ea4819c8e4521b752d1d82de11602 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/244ea4819c8e4521b752d1d82de11602 2024-12-06T06:14:13,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/192ffe9db3d5465789582046e27ca263 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/192ffe9db3d5465789582046e27ca263 2024-12-06T06:14:13,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a68b3fb71520459f9c10beced8846fcb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a68b3fb71520459f9c10beced8846fcb 2024-12-06T06:14:13,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/6c71a0c3dece4687bdee6f2149235b2c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/6c71a0c3dece4687bdee6f2149235b2c 2024-12-06T06:14:13,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8a7604e939c641f5afaa4a54677ab410 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8a7604e939c641f5afaa4a54677ab410 2024-12-06T06:14:13,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7657638d83b94390b15d547bc552d6d9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7657638d83b94390b15d547bc552d6d9 2024-12-06T06:14:13,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ea99e8af47da41ed84943f4177029237 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ea99e8af47da41ed84943f4177029237 2024-12-06T06:14:13,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/93d6c6b25c3e491b8cb8afabd769d370 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/93d6c6b25c3e491b8cb8afabd769d370 2024-12-06T06:14:13,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bf3bd24e7daa4747a0f3bc5b6d1d27a2 2024-12-06T06:14:13,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a9633258a7db49b3b3d21d8230a1b0b1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a9633258a7db49b3b3d21d8230a1b0b1 2024-12-06T06:14:13,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e3a326a45d7b4b209da9c3810a9efcff to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e3a326a45d7b4b209da9c3810a9efcff 2024-12-06T06:14:13,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd946af21f714263a77e8df7c2176a05 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/cd946af21f714263a77e8df7c2176a05 2024-12-06T06:14:13,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5db8ea3d7d2144b69430a7ad24ae22a1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5db8ea3d7d2144b69430a7ad24ae22a1 2024-12-06T06:14:13,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e53aeabf26e94efeb725a66f3ad5514f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/e53aeabf26e94efeb725a66f3ad5514f 2024-12-06T06:14:13,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/02e9ec369f6d43728c2032fb374579b1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/02e9ec369f6d43728c2032fb374579b1 2024-12-06T06:14:13,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f3113c5ea43448c1ad35c04ba9117924 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f3113c5ea43448c1ad35c04ba9117924 2024-12-06T06:14:13,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a82df4a5ad6a47ac9490c13848cf3b92 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/a82df4a5ad6a47ac9490c13848cf3b92 2024-12-06T06:14:13,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8fc31f5c48e64eb898ced3b9f9a765ba to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8fc31f5c48e64eb898ced3b9f9a765ba 2024-12-06T06:14:13,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/b455f7adde9243138f74c2b517c8b3ef to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/b455f7adde9243138f74c2b517c8b3ef 2024-12-06T06:14:13,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8aceaf431e00415cb2db730a1ed882f2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/8aceaf431e00415cb2db730a1ed882f2 2024-12-06T06:14:13,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ff7f2cee18f04278922f43515df84172 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/ff7f2cee18f04278922f43515df84172 2024-12-06T06:14:13,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7a73aaa34a2a4f89b4bf351d691a3450 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/7a73aaa34a2a4f89b4bf351d691a3450 2024-12-06T06:14:13,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/22b0411401bd41b184ae32842fcbb3c4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/22b0411401bd41b184ae32842fcbb3c4 2024-12-06T06:14:13,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bb09e525c81046fe9d7a307906a4255a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/bb09e525c81046fe9d7a307906a4255a 2024-12-06T06:14:13,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/652c5d91c5ad49e394c58619eb99ddae to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/652c5d91c5ad49e394c58619eb99ddae 2024-12-06T06:14:13,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d344c967927b4120bd22a72fa51d262a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d344c967927b4120bd22a72fa51d262a 2024-12-06T06:14:13,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/77ff2dcda37b44d8bd3a6f9db2a2883c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/77ff2dcda37b44d8bd3a6f9db2a2883c 2024-12-06T06:14:13,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/726697868f544a73a6fcb76eea75acfc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/726697868f544a73a6fcb76eea75acfc 2024-12-06T06:14:13,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/2fac4b2c7690498d9bb53d27e6bf0e18 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/2fac4b2c7690498d9bb53d27e6bf0e18 2024-12-06T06:14:13,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/dee15741e2c84a178ca9946072475950 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/dee15741e2c84a178ca9946072475950 2024-12-06T06:14:13,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d86faf96f98844ea91fe404c5a0edf47 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/d86faf96f98844ea91fe404c5a0edf47 2024-12-06T06:14:13,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80a65d435100477a8d48ab5f9f628c67 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80a65d435100477a8d48ab5f9f628c67 2024-12-06T06:14:13,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/30ee758e20ea40aea3bed31f6d4e2c72 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/30ee758e20ea40aea3bed31f6d4e2c72 2024-12-06T06:14:13,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80b96c02d272407fab8c93945fdb09b6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/80b96c02d272407fab8c93945fdb09b6 2024-12-06T06:14:13,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dc3da7853bc47658ab756d6e296ff16 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/4dc3da7853bc47658ab756d6e296ff16 2024-12-06T06:14:13,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5eae4b1f46ae4928bd5a820861ff7384, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b37fc7c43783471f89a8745d6b6a4872, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5ac56220f8514825924d12f6fdee3f45, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ab45337cdf4e48a8b25001f5cd5da2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3af5629cb3fc49eeab9b1cdd0a2b9615, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7df4809577f0499c9a6406a89dd2d90f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5875754dafd446cb8aa83c2038d469c3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/c9617a96b11847568ba1aa11966029e5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5450c4bb80814d5d83eab816b8721c7c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/100a8fc5cbb7474abfa3c0e00e904be4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/f141df63f02c43b7b3b1a1ca2c4d0a34, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/2efd0d52b6c74a1b91b28411b2e69c61, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/d439f82776f947ef8ebc967a2cd4ea0f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/70e4f0d1026746f7a1170fa443b5a7b4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/18e1155fb78d473a862a79a6e03602bb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/231f4db8f91b4ecb8b1410b5f49f2805, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/fe49e272b7804db0805ec0d610cba219, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/0c4d272675dd4b26b4adc3c730dbfa5d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7396d3d1c13a436d9fa78753ee2c1a9f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ddffeb5d184835bb018cae3f7190f4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/80169a8dbb5a4b18a2af081e6a2577e0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a223675fbfb2416db1ad29df1a7fcf10, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/8db2141e77d34ec4ad6198aa68c86404, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a1756085530745d5b7076dde374cb9cc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6d846a1692394375b6b0372b194c077f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3eaf5b6e82134a3295147758036055e9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e6f9541e0791457a883bf326c079907a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/138e4531347947699b1a09092a39340d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6ef6647b82544e81817153c32eb3d9f5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/bbeebd448f694c2a8ab4ffc244ef8061, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/79a6bf86facb4b86970d2bd4f14f1c1a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6033bf1255e94bc49f2ac46fd7cb3463, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e0e67472579d46c2aeb487a507484540, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b0a0e3a6ef3344cea32ea33c93674d9a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7ae880adf25942bf9113347b65fe37b7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/66529da7857c4290becdf71d90a227a8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/deb1ca81cb854fa29e97720c72734ddc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/91f8e46501ca4f75af741d25aa04bbdb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/dcf5da29726d4438b6a5a47d1a3ed71b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/eb877eb35a7844cc9b516575fec7405d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/461a7ebab2ea42b9a60e487d20d44f21, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/20003e4475b946cb92d21026399708f9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b44730a7147e4483b0f0c953d79ce8a1] to archive 2024-12-06T06:14:13,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:14:13,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5eae4b1f46ae4928bd5a820861ff7384 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5eae4b1f46ae4928bd5a820861ff7384 2024-12-06T06:14:13,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b37fc7c43783471f89a8745d6b6a4872 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b37fc7c43783471f89a8745d6b6a4872 2024-12-06T06:14:13,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5ac56220f8514825924d12f6fdee3f45 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5ac56220f8514825924d12f6fdee3f45 2024-12-06T06:14:13,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ab45337cdf4e48a8b25001f5cd5da2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ab45337cdf4e48a8b25001f5cd5da2 2024-12-06T06:14:13,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3af5629cb3fc49eeab9b1cdd0a2b9615 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3af5629cb3fc49eeab9b1cdd0a2b9615 2024-12-06T06:14:13,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7df4809577f0499c9a6406a89dd2d90f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7df4809577f0499c9a6406a89dd2d90f 2024-12-06T06:14:13,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5875754dafd446cb8aa83c2038d469c3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5875754dafd446cb8aa83c2038d469c3 2024-12-06T06:14:13,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/c9617a96b11847568ba1aa11966029e5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/c9617a96b11847568ba1aa11966029e5 2024-12-06T06:14:13,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5450c4bb80814d5d83eab816b8721c7c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5450c4bb80814d5d83eab816b8721c7c 2024-12-06T06:14:13,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/100a8fc5cbb7474abfa3c0e00e904be4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/100a8fc5cbb7474abfa3c0e00e904be4 2024-12-06T06:14:13,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/f141df63f02c43b7b3b1a1ca2c4d0a34 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/f141df63f02c43b7b3b1a1ca2c4d0a34 2024-12-06T06:14:13,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/2efd0d52b6c74a1b91b28411b2e69c61 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/2efd0d52b6c74a1b91b28411b2e69c61 2024-12-06T06:14:13,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/d439f82776f947ef8ebc967a2cd4ea0f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/d439f82776f947ef8ebc967a2cd4ea0f 2024-12-06T06:14:13,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/70e4f0d1026746f7a1170fa443b5a7b4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/70e4f0d1026746f7a1170fa443b5a7b4 2024-12-06T06:14:13,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/18e1155fb78d473a862a79a6e03602bb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/18e1155fb78d473a862a79a6e03602bb 2024-12-06T06:14:13,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/231f4db8f91b4ecb8b1410b5f49f2805 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/231f4db8f91b4ecb8b1410b5f49f2805 2024-12-06T06:14:13,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/fe49e272b7804db0805ec0d610cba219 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/fe49e272b7804db0805ec0d610cba219 2024-12-06T06:14:13,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/0c4d272675dd4b26b4adc3c730dbfa5d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/0c4d272675dd4b26b4adc3c730dbfa5d 2024-12-06T06:14:13,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7396d3d1c13a436d9fa78753ee2c1a9f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7396d3d1c13a436d9fa78753ee2c1a9f 2024-12-06T06:14:13,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ddffeb5d184835bb018cae3f7190f4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/03ddffeb5d184835bb018cae3f7190f4 2024-12-06T06:14:13,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/80169a8dbb5a4b18a2af081e6a2577e0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/80169a8dbb5a4b18a2af081e6a2577e0 2024-12-06T06:14:13,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a223675fbfb2416db1ad29df1a7fcf10 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a223675fbfb2416db1ad29df1a7fcf10 2024-12-06T06:14:13,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/8db2141e77d34ec4ad6198aa68c86404 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/8db2141e77d34ec4ad6198aa68c86404 2024-12-06T06:14:13,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a1756085530745d5b7076dde374cb9cc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/a1756085530745d5b7076dde374cb9cc 2024-12-06T06:14:13,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6d846a1692394375b6b0372b194c077f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6d846a1692394375b6b0372b194c077f 2024-12-06T06:14:13,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3eaf5b6e82134a3295147758036055e9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/3eaf5b6e82134a3295147758036055e9 2024-12-06T06:14:13,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e6f9541e0791457a883bf326c079907a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e6f9541e0791457a883bf326c079907a 2024-12-06T06:14:13,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/138e4531347947699b1a09092a39340d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/138e4531347947699b1a09092a39340d 2024-12-06T06:14:13,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6ef6647b82544e81817153c32eb3d9f5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6ef6647b82544e81817153c32eb3d9f5 2024-12-06T06:14:13,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/bbeebd448f694c2a8ab4ffc244ef8061 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/bbeebd448f694c2a8ab4ffc244ef8061 2024-12-06T06:14:13,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/79a6bf86facb4b86970d2bd4f14f1c1a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/79a6bf86facb4b86970d2bd4f14f1c1a 2024-12-06T06:14:13,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6033bf1255e94bc49f2ac46fd7cb3463 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/6033bf1255e94bc49f2ac46fd7cb3463 2024-12-06T06:14:13,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e0e67472579d46c2aeb487a507484540 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/e0e67472579d46c2aeb487a507484540 2024-12-06T06:14:13,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b0a0e3a6ef3344cea32ea33c93674d9a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b0a0e3a6ef3344cea32ea33c93674d9a 2024-12-06T06:14:13,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7ae880adf25942bf9113347b65fe37b7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/7ae880adf25942bf9113347b65fe37b7 2024-12-06T06:14:13,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/66529da7857c4290becdf71d90a227a8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/66529da7857c4290becdf71d90a227a8 2024-12-06T06:14:13,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/deb1ca81cb854fa29e97720c72734ddc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/deb1ca81cb854fa29e97720c72734ddc 2024-12-06T06:14:13,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/91f8e46501ca4f75af741d25aa04bbdb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/91f8e46501ca4f75af741d25aa04bbdb 2024-12-06T06:14:13,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/dcf5da29726d4438b6a5a47d1a3ed71b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/dcf5da29726d4438b6a5a47d1a3ed71b 2024-12-06T06:14:13,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/eb877eb35a7844cc9b516575fec7405d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/eb877eb35a7844cc9b516575fec7405d 2024-12-06T06:14:13,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/461a7ebab2ea42b9a60e487d20d44f21 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/461a7ebab2ea42b9a60e487d20d44f21 2024-12-06T06:14:13,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/20003e4475b946cb92d21026399708f9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/20003e4475b946cb92d21026399708f9 2024-12-06T06:14:13,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b44730a7147e4483b0f0c953d79ce8a1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/b44730a7147e4483b0f0c953d79ce8a1 2024-12-06T06:14:13,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ae4b8c75123d4d909ea0fcf7638cf3ef, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/aa025e7e281b4b4f86e00b6cfd7fb2f1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/483a7bc299374804bbc19b5b93e340e4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e58d25b7cbc24b8dac415aaf04ef6ecd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8033da6b873945df93089eba6645bba0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/5e3bbdb84ba748f194fb682ea401cb7f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e7e2726d84f4e0d9888a14e3e2c25d5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f47ad99e4ee8477abb4423407f6476c8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ee2770db77a644faa64dc5cd0f92c672, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/c67d6287ec2648afa6fb9681f80f0c99, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b9c5d58f9bde4286973bc2a0d883c409, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/68ef9bd3e4014e7fa494ce171c554e66, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b619c297c5b14700ba724bfc73852680, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/d20425bfc169460ab0e324c35b102d73, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e742c8dd10e4cf7aa0ca210a63eb33d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/64d14fc0a8ec4256be5b75322bf47e2b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e0fe639a36414e20ba9b91a463a1c4b3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90d7d0bf77384c62bc84aea4df30c460, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f75d0f4fa5c7481295b89b19bc437750, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/54b3a749018447c7835b4d860a72453b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cff709e8108944ff92ae70e10f3e43a1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9da458e7fcca4033a79e8bedca558948, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8b65e4bb3bbf4b408dc60b21386c7ff0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/846c3019add040b9bdf0d0380ecab78b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/66a76128591d4ae6bc327de0e1c209fb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/18b6680992b8456ca9702124f9e23e6a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cf0aff44529b44b59449522de3ecfb27, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1fcc2bac0d9b428f8334d498826eade4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/42ee6f9fe6d74c08b215ce0a4444e566, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/85ed08afb467417e946053f7afe08086, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dc81612b56f54713b02704bb50e2b21a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b79eb3c93bb04f31922e3848ff446eb2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/6d180e697bac48d5a869f85f0956e6d7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/333efedad9694b748d16167d3c162150, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/60ed9ef0aed94baa8c758b7a4f535ee6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e7f70980bcaa4362b11ecf85ecf980e8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/921904a4ea924759a9b6b9cb775d9b72, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/58f4fb2bcf434d1b8dc1b7dd78cb0447, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/3f6b616ccf3946259b5eb2f6b74d8aaa, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b43b0411c92941a5973831c10f1433e2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/550b5dc9717e436089057e1701ec261e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b073d80de0924c6a880b8b802ba3ec1e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/91752b6deb7b4bcd831be8762be7d981, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dd3d20afd51749d79d2986fcb94392f0] to archive 2024-12-06T06:14:13,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:14:13,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ae4b8c75123d4d909ea0fcf7638cf3ef to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ae4b8c75123d4d909ea0fcf7638cf3ef 2024-12-06T06:14:13,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/aa025e7e281b4b4f86e00b6cfd7fb2f1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/aa025e7e281b4b4f86e00b6cfd7fb2f1 2024-12-06T06:14:13,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/483a7bc299374804bbc19b5b93e340e4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/483a7bc299374804bbc19b5b93e340e4 2024-12-06T06:14:13,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e58d25b7cbc24b8dac415aaf04ef6ecd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e58d25b7cbc24b8dac415aaf04ef6ecd 2024-12-06T06:14:13,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8033da6b873945df93089eba6645bba0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8033da6b873945df93089eba6645bba0 2024-12-06T06:14:13,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/5e3bbdb84ba748f194fb682ea401cb7f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/5e3bbdb84ba748f194fb682ea401cb7f 2024-12-06T06:14:13,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e7e2726d84f4e0d9888a14e3e2c25d5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e7e2726d84f4e0d9888a14e3e2c25d5 2024-12-06T06:14:13,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f47ad99e4ee8477abb4423407f6476c8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f47ad99e4ee8477abb4423407f6476c8 2024-12-06T06:14:13,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ee2770db77a644faa64dc5cd0f92c672 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/ee2770db77a644faa64dc5cd0f92c672 2024-12-06T06:14:13,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/c67d6287ec2648afa6fb9681f80f0c99 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/c67d6287ec2648afa6fb9681f80f0c99 2024-12-06T06:14:13,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b9c5d58f9bde4286973bc2a0d883c409 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b9c5d58f9bde4286973bc2a0d883c409 2024-12-06T06:14:13,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/68ef9bd3e4014e7fa494ce171c554e66 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/68ef9bd3e4014e7fa494ce171c554e66 2024-12-06T06:14:13,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b619c297c5b14700ba724bfc73852680 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b619c297c5b14700ba724bfc73852680 2024-12-06T06:14:13,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/d20425bfc169460ab0e324c35b102d73 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/d20425bfc169460ab0e324c35b102d73 2024-12-06T06:14:13,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e742c8dd10e4cf7aa0ca210a63eb33d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1e742c8dd10e4cf7aa0ca210a63eb33d 2024-12-06T06:14:13,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/64d14fc0a8ec4256be5b75322bf47e2b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/64d14fc0a8ec4256be5b75322bf47e2b 2024-12-06T06:14:13,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e0fe639a36414e20ba9b91a463a1c4b3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e0fe639a36414e20ba9b91a463a1c4b3 2024-12-06T06:14:13,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90d7d0bf77384c62bc84aea4df30c460 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90d7d0bf77384c62bc84aea4df30c460 2024-12-06T06:14:13,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f75d0f4fa5c7481295b89b19bc437750 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/f75d0f4fa5c7481295b89b19bc437750 2024-12-06T06:14:13,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/54b3a749018447c7835b4d860a72453b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/54b3a749018447c7835b4d860a72453b 2024-12-06T06:14:13,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cff709e8108944ff92ae70e10f3e43a1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cff709e8108944ff92ae70e10f3e43a1 2024-12-06T06:14:13,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9da458e7fcca4033a79e8bedca558948 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9da458e7fcca4033a79e8bedca558948 2024-12-06T06:14:13,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8b65e4bb3bbf4b408dc60b21386c7ff0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/8b65e4bb3bbf4b408dc60b21386c7ff0 2024-12-06T06:14:13,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/846c3019add040b9bdf0d0380ecab78b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/846c3019add040b9bdf0d0380ecab78b 2024-12-06T06:14:13,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/66a76128591d4ae6bc327de0e1c209fb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/66a76128591d4ae6bc327de0e1c209fb 2024-12-06T06:14:13,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/18b6680992b8456ca9702124f9e23e6a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/18b6680992b8456ca9702124f9e23e6a 2024-12-06T06:14:13,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cf0aff44529b44b59449522de3ecfb27 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/cf0aff44529b44b59449522de3ecfb27 2024-12-06T06:14:13,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1fcc2bac0d9b428f8334d498826eade4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/1fcc2bac0d9b428f8334d498826eade4 2024-12-06T06:14:13,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/42ee6f9fe6d74c08b215ce0a4444e566 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/42ee6f9fe6d74c08b215ce0a4444e566 2024-12-06T06:14:13,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/85ed08afb467417e946053f7afe08086 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/85ed08afb467417e946053f7afe08086 2024-12-06T06:14:13,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dc81612b56f54713b02704bb50e2b21a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dc81612b56f54713b02704bb50e2b21a 2024-12-06T06:14:13,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b79eb3c93bb04f31922e3848ff446eb2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b79eb3c93bb04f31922e3848ff446eb2 2024-12-06T06:14:13,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/6d180e697bac48d5a869f85f0956e6d7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/6d180e697bac48d5a869f85f0956e6d7 2024-12-06T06:14:13,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/333efedad9694b748d16167d3c162150 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/333efedad9694b748d16167d3c162150 2024-12-06T06:14:13,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/60ed9ef0aed94baa8c758b7a4f535ee6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/60ed9ef0aed94baa8c758b7a4f535ee6 2024-12-06T06:14:13,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e7f70980bcaa4362b11ecf85ecf980e8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/e7f70980bcaa4362b11ecf85ecf980e8 2024-12-06T06:14:13,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/921904a4ea924759a9b6b9cb775d9b72 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/921904a4ea924759a9b6b9cb775d9b72 2024-12-06T06:14:13,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/58f4fb2bcf434d1b8dc1b7dd78cb0447 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/58f4fb2bcf434d1b8dc1b7dd78cb0447 2024-12-06T06:14:13,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/3f6b616ccf3946259b5eb2f6b74d8aaa to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/3f6b616ccf3946259b5eb2f6b74d8aaa 2024-12-06T06:14:13,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b43b0411c92941a5973831c10f1433e2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b43b0411c92941a5973831c10f1433e2 2024-12-06T06:14:13,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/550b5dc9717e436089057e1701ec261e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/550b5dc9717e436089057e1701ec261e 2024-12-06T06:14:13,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b073d80de0924c6a880b8b802ba3ec1e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/b073d80de0924c6a880b8b802ba3ec1e 2024-12-06T06:14:13,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/91752b6deb7b4bcd831be8762be7d981 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/91752b6deb7b4bcd831be8762be7d981 2024-12-06T06:14:13,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dd3d20afd51749d79d2986fcb94392f0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/dd3d20afd51749d79d2986fcb94392f0 2024-12-06T06:14:13,158 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/recovered.edits/752.seqid, newMaxSeqId=752, maxSeqId=1 2024-12-06T06:14:13,158 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. 2024-12-06T06:14:13,158 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 3860e7c2497c4799e160cda168e0086d: Waiting for close lock at 1733465651736Running coprocessor pre-close hooks at 1733465651736Disabling compacts and flushes for region at 1733465651736Disabling writes for close at 1733465651736Obtaining lock to block concurrent updates at 1733465651736Preparing flush snapshotting stores in 3860e7c2497c4799e160cda168e0086d at 1733465651736Finished memstore snapshotting TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733465651737 (+1 ms)Flushing stores of TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d. at 1733465651737Flushing 3860e7c2497c4799e160cda168e0086d/A: creating writer at 1733465651737Flushing 3860e7c2497c4799e160cda168e0086d/A: appending metadata at 1733465651740 (+3 ms)Flushing 3860e7c2497c4799e160cda168e0086d/A: closing flushed file at 1733465651740Flushing 3860e7c2497c4799e160cda168e0086d/B: creating writer at 1733465652150 (+410 ms)Flushing 3860e7c2497c4799e160cda168e0086d/B: appending metadata at 1733465652152 (+2 ms)Flushing 3860e7c2497c4799e160cda168e0086d/B: closing flushed file at 1733465652152Flushing 3860e7c2497c4799e160cda168e0086d/C: creating writer at 1733465652561 (+409 ms)Flushing 3860e7c2497c4799e160cda168e0086d/C: appending metadata at 1733465652564 (+3 ms)Flushing 3860e7c2497c4799e160cda168e0086d/C: closing flushed file at 1733465652564Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bd9fc2c: reopening flushed file at 1733465652973 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46b0b373: reopening flushed file at 1733465652977 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@388efc5a: reopening flushed file at 1733465652981 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 3860e7c2497c4799e160cda168e0086d in 1251ms, sequenceid=749, compaction requested=true at 1733465652987 (+6 ms)Writing region close event to WAL at 1733465653154 (+167 ms)Running coprocessor post-close hooks at 1733465653158 (+4 ms)Closed at 1733465653158 2024-12-06T06:14:13,161 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:13,161 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3860e7c2497c4799e160cda168e0086d, regionState=CLOSED 2024-12-06T06:14:13,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:14:13,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-06T06:14:13,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 3860e7c2497c4799e160cda168e0086d, server=6336e8474e1c,39987,1733465620995 in 1.5850 sec 2024-12-06T06:14:13,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-06T06:14:13,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3860e7c2497c4799e160cda168e0086d, UNASSIGN in 1.5910 sec 2024-12-06T06:14:13,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-06T06:14:13,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5970 sec 2024-12-06T06:14:13,170 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465653170"}]},"ts":"1733465653170"} 2024-12-06T06:14:13,171 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T06:14:13,171 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T06:14:13,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6100 sec 2024-12-06T06:14:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T06:14:13,715 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:13,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T06:14:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:13,723 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-06T06:14:13,725 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:13,729 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:13,733 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/recovered.edits] 2024-12-06T06:14:13,736 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/227c760ea76f4b288285ffa7077a8331 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/227c760ea76f4b288285ffa7077a8331 2024-12-06T06:14:13,737 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5a37211820004a5bab24b539e5f087e3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/5a37211820004a5bab24b539e5f087e3 2024-12-06T06:14:13,738 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f71c56d6ab92445aad9fcc896911e766 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/A/f71c56d6ab92445aad9fcc896911e766 2024-12-06T06:14:13,741 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/50b481e6feac4e42932e0ef1722820d2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/50b481e6feac4e42932e0ef1722820d2 2024-12-06T06:14:13,742 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5b85b6566a6d46468f2a19164910ed3d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/5b85b6566a6d46468f2a19164910ed3d 2024-12-06T06:14:13,743 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/9128d85565204a62ab4bc4091cd85a26 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/B/9128d85565204a62ab4bc4091cd85a26 2024-12-06T06:14:13,745 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9026a0d7abd14d05b3f245c5e14d57f1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9026a0d7abd14d05b3f245c5e14d57f1 2024-12-06T06:14:13,746 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90a0bf76258b44c1b669ec592217b6f4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/90a0bf76258b44c1b669ec592217b6f4 2024-12-06T06:14:13,747 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9213a8d814984abfb206c7fb6cb7a3fa to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/C/9213a8d814984abfb206c7fb6cb7a3fa 2024-12-06T06:14:13,750 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/recovered.edits/752.seqid to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d/recovered.edits/752.seqid 2024-12-06T06:14:13,750 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/3860e7c2497c4799e160cda168e0086d 2024-12-06T06:14:13,751 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T06:14:13,756 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:13,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-06T06:14:13,764 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T06:14:13,794 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T06:14:13,796 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:13,796 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T06:14:13,796 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733465653796"}]},"ts":"9223372036854775807"} 2024-12-06T06:14:13,800 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T06:14:13,800 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3860e7c2497c4799e160cda168e0086d, NAME => 'TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T06:14:13,800 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T06:14:13,801 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733465653800"}]},"ts":"9223372036854775807"} 2024-12-06T06:14:13,803 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T06:14:13,804 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:13,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 86 msec 2024-12-06T06:14:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-06T06:14:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T06:14:13,834 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:13,847 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869409647_22 at /127.0.0.1:57684 [Waiting for operation #424] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;6336e8474e1c:39987-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2037682797_22 at /127.0.0.1:57560 [Waiting for operation #437] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=388 (was 285) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5515 (was 5332) - AvailableMemoryMB LEAK? - 2024-12-06T06:14:13,857 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=388, ProcessCount=11, AvailableMemoryMB=5514 2024-12-06T06:14:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:14:13,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:14:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:13,861 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T06:14:13,861 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:13,861 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-06T06:14:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T06:14:13,862 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T06:14:13,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741977_1153 (size=1061) 2024-12-06T06:14:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T06:14:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T06:14:14,270 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea 2024-12-06T06:14:14,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741978_1154 (size=53) 2024-12-06T06:14:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T06:14:14,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:14:14,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing efacad63d89edd72fefc481a1ce9f017, disabling compactions & flushes 2024-12-06T06:14:14,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:14,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:14,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. after waiting 0 ms 2024-12-06T06:14:14,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:14,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:14,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for efacad63d89edd72fefc481a1ce9f017: Waiting for close lock at 1733465654676Disabling compacts and flushes for region at 1733465654676Disabling writes for close at 1733465654676Writing region close event to WAL at 1733465654676Closed at 1733465654676 2024-12-06T06:14:14,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T06:14:14,678 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733465654678"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733465654678"}]},"ts":"1733465654678"} 2024-12-06T06:14:14,680 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T06:14:14,681 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T06:14:14,681 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465654681"}]},"ts":"1733465654681"} 2024-12-06T06:14:14,682 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T06:14:14,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, ASSIGN}] 2024-12-06T06:14:14,683 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, ASSIGN 2024-12-06T06:14:14,684 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, ASSIGN; state=OFFLINE, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=false 2024-12-06T06:14:14,835 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efacad63d89edd72fefc481a1ce9f017, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:14,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, ASSIGN because future has completed 2024-12-06T06:14:14,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:14:14,993 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:14,993 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:14:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T06:14:14,994 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:14,994 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:14:14,994 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:14,994 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:14,995 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:14,996 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:14,997 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efacad63d89edd72fefc481a1ce9f017 columnFamilyName A 2024-12-06T06:14:14,997 DEBUG [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:14,998 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(327): Store=efacad63d89edd72fefc481a1ce9f017/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:14,998 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:14,999 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:14,999 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efacad63d89edd72fefc481a1ce9f017 columnFamilyName B 2024-12-06T06:14:14,999 DEBUG [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:15,000 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(327): Store=efacad63d89edd72fefc481a1ce9f017/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:15,000 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:15,001 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:15,001 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efacad63d89edd72fefc481a1ce9f017 columnFamilyName C 2024-12-06T06:14:15,001 DEBUG [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:15,001 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(327): Store=efacad63d89edd72fefc481a1ce9f017/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:15,001 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:15,001 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:15,002 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:15,003 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:15,004 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:15,004 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:15,004 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:14:15,005 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:15,007 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T06:14:15,008 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened efacad63d89edd72fefc481a1ce9f017; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70482358, jitterRate=0.050268977880477905}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:14:15,008 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:15,009 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for efacad63d89edd72fefc481a1ce9f017: Running coprocessor pre-open hook at 1733465654994Writing region info on filesystem at 1733465654994Initializing all the Stores at 1733465654995 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465654995Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465654995Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465654995Cleaning up temporary data from old regions at 1733465655004 (+9 ms)Running coprocessor post-open hooks at 1733465655008 (+4 ms)Region opened successfully at 1733465655009 (+1 ms) 2024-12-06T06:14:15,010 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., pid=36, masterSystemTime=1733465654990 2024-12-06T06:14:15,012 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:15,012 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:15,012 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efacad63d89edd72fefc481a1ce9f017, regionState=OPEN, openSeqNum=2, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:15,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:14:15,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-06T06:14:15,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 in 178 msec 2024-12-06T06:14:15,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-06T06:14:15,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, ASSIGN in 333 msec 2024-12-06T06:14:15,019 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T06:14:15,019 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465655019"}]},"ts":"1733465655019"} 2024-12-06T06:14:15,020 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T06:14:15,021 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T06:14:15,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-12-06T06:14:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T06:14:16,014 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:16,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65258394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:16,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:16,017 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5109dd68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:16,019 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251113c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,020 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:16,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:16,024 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T06:14:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 6336e8474e1c,40959,1733465620259 2024-12-06T06:14:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e2769e 2024-12-06T06:14:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T06:14:16,026 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T06:14:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:14:16,032 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:14:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:16,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741979_1155 (size=1097) 2024-12-06T06:14:16,454 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-06T06:14:16,454 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-06T06:14:16,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:14:16,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, REOPEN/MOVE}] 2024-12-06T06:14:16,465 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, REOPEN/MOVE 2024-12-06T06:14:16,465 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=efacad63d89edd72fefc481a1ce9f017, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:16,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, REOPEN/MOVE because future has completed 2024-12-06T06:14:16,467 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:14:16,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:14:16,620 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,620 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:14:16,620 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing efacad63d89edd72fefc481a1ce9f017, disabling compactions & flushes 2024-12-06T06:14:16,620 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:16,620 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:16,620 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. after waiting 0 ms 2024-12-06T06:14:16,620 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:16,624 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T06:14:16,625 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:16,625 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for efacad63d89edd72fefc481a1ce9f017: Waiting for close lock at 1733465656620Running coprocessor pre-close hooks at 1733465656620Disabling compacts and flushes for region at 1733465656620Disabling writes for close at 1733465656620Writing region close event to WAL at 1733465656621 (+1 ms)Running coprocessor post-close hooks at 1733465656625 (+4 ms)Closed at 1733465656625 2024-12-06T06:14:16,625 WARN [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: efacad63d89edd72fefc481a1ce9f017 to self. 2024-12-06T06:14:16,627 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,627 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=efacad63d89edd72fefc481a1ce9f017, regionState=CLOSED 2024-12-06T06:14:16,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:14:16,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-06T06:14:16,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 in 163 msec 2024-12-06T06:14:16,632 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, REOPEN/MOVE; state=CLOSED, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=true 2024-12-06T06:14:16,782 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=efacad63d89edd72fefc481a1ce9f017, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:16,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, REOPEN/MOVE because future has completed 2024-12-06T06:14:16,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:14:16,939 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:16,940 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:14:16,940 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,940 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:14:16,940 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,940 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,944 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,945 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:16,950 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efacad63d89edd72fefc481a1ce9f017 columnFamilyName A 2024-12-06T06:14:16,952 DEBUG [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:16,952 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(327): Store=efacad63d89edd72fefc481a1ce9f017/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:16,953 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,953 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:16,953 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efacad63d89edd72fefc481a1ce9f017 columnFamilyName B 2024-12-06T06:14:16,954 DEBUG [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:16,954 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(327): Store=efacad63d89edd72fefc481a1ce9f017/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:16,954 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,955 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:16,955 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efacad63d89edd72fefc481a1ce9f017 columnFamilyName C 2024-12-06T06:14:16,955 DEBUG [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:16,955 INFO [StoreOpener-efacad63d89edd72fefc481a1ce9f017-1 {}] regionserver.HStore(327): Store=efacad63d89edd72fefc481a1ce9f017/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:16,955 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:16,955 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,956 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,957 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,958 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,958 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,958 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:14:16,959 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,960 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened efacad63d89edd72fefc481a1ce9f017; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64066375, jitterRate=-0.04533661901950836}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:14:16,960 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:16,961 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for efacad63d89edd72fefc481a1ce9f017: Running coprocessor pre-open hook at 1733465656940Writing region info on filesystem at 1733465656940Initializing all the Stores at 1733465656941 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465656941Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465656941Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465656941Cleaning up temporary data from old regions at 1733465656958 (+17 ms)Running coprocessor post-open hooks at 1733465656960 (+2 ms)Region opened successfully at 1733465656961 (+1 ms) 2024-12-06T06:14:16,962 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., pid=41, masterSystemTime=1733465656936 2024-12-06T06:14:16,964 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:16,964 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:16,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=efacad63d89edd72fefc481a1ce9f017, regionState=OPEN, openSeqNum=5, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:16,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:14:16,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-06T06:14:16,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 in 182 msec 2024-12-06T06:14:16,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-06T06:14:16,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, REOPEN/MOVE in 505 msec 2024-12-06T06:14:16,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-06T06:14:16,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-12-06T06:14:16,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-12-06T06:14:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-06T06:14:16,978 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:16,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:16,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:16,980 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a862a31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:16,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7410743f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:16,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:16,984 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7603d748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:16,986 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a920a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:16,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:16,988 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c190689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:16,989 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d3321b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:16,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:16,991 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eeee101, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:16,992 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a95274b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:16,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:16,994 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3051b641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:16,995 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6e6c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:16,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:16,997 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57867419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:16,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@414e4bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:16,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:16,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:17,000 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d13d6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:17,001 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb39de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:17,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2874ba29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:17,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:17,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:17,003 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:17,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:17,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da5b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:17,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:17,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:17,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:17,005 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bda06d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:17,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139bfa79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:17,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:17,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:17,007 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d62886d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:17,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:17,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:17,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:17,009 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c8aab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:17,010 DEBUG [Thread-767 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,010 DEBUG [Thread-752 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,010 DEBUG [Thread-763 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,011 DEBUG [Thread-750 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,011 DEBUG [Thread-754 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,011 DEBUG [Thread-758 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,014 DEBUG [Thread-756 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,015 DEBUG [Thread-765 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-06T06:14:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T06:14:17,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:17,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:17,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:17,017 DEBUG [Thread-761 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:17,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:17,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:17,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:14:17,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:14:17,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:17,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b157f8f578824a8d96c2349c3af8a2bd_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465657034/Put/seqid=0 2024-12-06T06:14:17,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741980_1156 (size=14594) 2024-12-06T06:14:17,081 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:17,087 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b157f8f578824a8d96c2349c3af8a2bd_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b157f8f578824a8d96c2349c3af8a2bd_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:17,089 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/a2d1bd685ec540358cd59ddccf32d280, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:17,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/a2d1bd685ec540358cd59ddccf32d280 is 175, key is test_row_0/A:col10/1733465657034/Put/seqid=0 2024-12-06T06:14:17,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741981_1157 (size=39549) 2024-12-06T06:14:17,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/a2d1bd685ec540358cd59ddccf32d280 2024-12-06T06:14:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T06:14:17,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733465657098, max timestamp=1733465657130Num uniques 859; 2024-12-06T06:14:17,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:17,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465657098, max timestamp=1733465657130Num uniques 859; 2024-12-06T06:14:17,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:17,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465657098, max timestamp=1733465657130Num uniques 859; 2024-12-06T06:14:17,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:17,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/a5f9d45634744f6d8ef29284d227dbc4 is 50, key is test_row_0/B:col10/1733465657034/Put/seqid=0 2024-12-06T06:14:17,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:17,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741982_1158 (size=12001) 2024-12-06T06:14:17,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/a5f9d45634744f6d8ef29284d227dbc4 2024-12-06T06:14:17,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465717177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465717180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465717182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465717185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465717188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/d8636115608e4357a5a8bccac9ce779a is 50, key is test_row_0/C:col10/1733465657034/Put/seqid=0 2024-12-06T06:14:17,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741983_1159 (size=12001) 2024-12-06T06:14:17,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/d8636115608e4357a5a8bccac9ce779a 2024-12-06T06:14:17,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/a2d1bd685ec540358cd59ddccf32d280 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/a2d1bd685ec540358cd59ddccf32d280 2024-12-06T06:14:17,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/a2d1bd685ec540358cd59ddccf32d280, entries=200, sequenceid=14, filesize=38.6 K 2024-12-06T06:14:17,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/a5f9d45634744f6d8ef29284d227dbc4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/a5f9d45634744f6d8ef29284d227dbc4 2024-12-06T06:14:17,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/a5f9d45634744f6d8ef29284d227dbc4, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T06:14:17,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/d8636115608e4357a5a8bccac9ce779a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d8636115608e4357a5a8bccac9ce779a 2024-12-06T06:14:17,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d8636115608e4357a5a8bccac9ce779a, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T06:14:17,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for efacad63d89edd72fefc481a1ce9f017 in 219ms, sequenceid=14, compaction requested=false 2024-12-06T06:14:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T06:14:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:17,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=140.89 KB heapSize=400.45 KB 2024-12-06T06:14:17,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733465657177, max timestamp=1733465657310Num uniques 1068; 2024-12-06T06:14:17,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-06T06:14:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:14:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:14:17,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T06:14:17,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206eace28d72dc64caf9636747ec30fb533_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465657184/Put/seqid=0 2024-12-06T06:14:17,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741984_1160 (size=12154) 2024-12-06T06:14:17,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465717377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465717421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465717421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465717421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465717428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465717544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465717564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465717565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465717565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465717565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:17,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:17,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T06:14:17,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:17,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465717754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,758 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206eace28d72dc64caf9636747ec30fb533_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eace28d72dc64caf9636747ec30fb533_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:17,759 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fa7de25415ce4129b15726d6b9d179f8, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:17,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fa7de25415ce4129b15726d6b9d179f8 is 175, key is test_row_0/A:col10/1733465657184/Put/seqid=0 2024-12-06T06:14:17,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741985_1161 (size=30955) 2024-12-06T06:14:17,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fa7de25415ce4129b15726d6b9d179f8 2024-12-06T06:14:17,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465717777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465717777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465717778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465717778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:17,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d2781b4a56ef4ed193c325383d450d12 is 50, key is test_row_0/B:col10/1733465657184/Put/seqid=0 2024-12-06T06:14:17,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741986_1162 (size=12001) 2024-12-06T06:14:17,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:17,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:17,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d2781b4a56ef4ed193c325383d450d12 2024-12-06T06:14:17,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b9bcf146afda4cd199f68b28f67e466d is 50, key is test_row_0/C:col10/1733465657184/Put/seqid=0 2024-12-06T06:14:17,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741987_1163 (size=12001) 2024-12-06T06:14:17,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:17,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:18,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465718064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465718083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465718083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465718083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465718095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:18,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:18,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T06:14:18,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b9bcf146afda4cd199f68b28f67e466d 2024-12-06T06:14:18,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fa7de25415ce4129b15726d6b9d179f8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa7de25415ce4129b15726d6b9d179f8 2024-12-06T06:14:18,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa7de25415ce4129b15726d6b9d179f8, entries=150, sequenceid=52, filesize=30.2 K 2024-12-06T06:14:18,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d2781b4a56ef4ed193c325383d450d12 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d2781b4a56ef4ed193c325383d450d12 2024-12-06T06:14:18,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d2781b4a56ef4ed193c325383d450d12, entries=150, sequenceid=52, filesize=11.7 K 2024-12-06T06:14:18,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b9bcf146afda4cd199f68b28f67e466d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b9bcf146afda4cd199f68b28f67e466d 2024-12-06T06:14:18,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:18,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:18,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:18,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:18,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:18,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:18,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b9bcf146afda4cd199f68b28f67e466d, entries=150, sequenceid=52, filesize=11.7 K 2024-12-06T06:14:18,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.58 KB/112210, heapSize ~307.39 KB/314768, currentSize=73.80 KB/75570 for efacad63d89edd72fefc481a1ce9f017 in 950ms, sequenceid=52, compaction requested=false 2024-12-06T06:14:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:18,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T06:14:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:18,413 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T06:14:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:18,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120688185c3820aa49f1a75ceac97829f55f_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465657420/Put/seqid=0 2024-12-06T06:14:18,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741988_1164 (size=12154) 2024-12-06T06:14:18,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:18,450 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120688185c3820aa49f1a75ceac97829f55f_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120688185c3820aa49f1a75ceac97829f55f_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:18,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/9ff6993d1b30492db2c2fc84acbea254, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:18,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/9ff6993d1b30492db2c2fc84acbea254 is 175, key is test_row_0/A:col10/1733465657420/Put/seqid=0 2024-12-06T06:14:18,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741989_1165 (size=30955) 2024-12-06T06:14:18,465 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=66, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/9ff6993d1b30492db2c2fc84acbea254 2024-12-06T06:14:18,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/69ac9f6db4564f7fa25f3317628912d2 is 50, key is test_row_0/B:col10/1733465657420/Put/seqid=0 2024-12-06T06:14:18,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741990_1166 (size=12001) 2024-12-06T06:14:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:18,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465718609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465718610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465718611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465718612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465718613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,695 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T06:14:18,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465718726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465718726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465718726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465718726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465718726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,894 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/69ac9f6db4564f7fa25f3317628912d2 2024-12-06T06:14:18,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/06c1249c8faa49d0b90f9ad341c58834 is 50, key is test_row_0/C:col10/1733465657420/Put/seqid=0 2024-12-06T06:14:18,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741991_1167 (size=12001) 2024-12-06T06:14:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465718944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465718944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465718944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465718945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465718945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T06:14:19,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465719254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465719254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465719254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465719263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465719263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,309 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/06c1249c8faa49d0b90f9ad341c58834 2024-12-06T06:14:19,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/9ff6993d1b30492db2c2fc84acbea254 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9ff6993d1b30492db2c2fc84acbea254 2024-12-06T06:14:19,319 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9ff6993d1b30492db2c2fc84acbea254, entries=150, sequenceid=66, filesize=30.2 K 2024-12-06T06:14:19,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/69ac9f6db4564f7fa25f3317628912d2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/69ac9f6db4564f7fa25f3317628912d2 2024-12-06T06:14:19,324 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/69ac9f6db4564f7fa25f3317628912d2, entries=150, sequenceid=66, filesize=11.7 K 2024-12-06T06:14:19,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/06c1249c8faa49d0b90f9ad341c58834 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/06c1249c8faa49d0b90f9ad341c58834 2024-12-06T06:14:19,331 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/06c1249c8faa49d0b90f9ad341c58834, entries=150, sequenceid=66, filesize=11.7 K 2024-12-06T06:14:19,332 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for efacad63d89edd72fefc481a1ce9f017 in 919ms, sequenceid=66, compaction requested=true 2024-12-06T06:14:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-06T06:14:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-06T06:14:19,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-06T06:14:19,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3170 sec 2024-12-06T06:14:19,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3240 sec 2024-12-06T06:14:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:19,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:14:19,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:19,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:19,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:19,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:19,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:19,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:19,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206215c1f7a55ce4faabf0ab5d165574868_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465658610/Put/seqid=0 2024-12-06T06:14:19,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741992_1168 (size=14594) 2024-12-06T06:14:19,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465719780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465719781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465719782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465719783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465719783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465719894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465719894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465719894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465719894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465720103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465720104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465720104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465720114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,186 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:20,191 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206215c1f7a55ce4faabf0ab5d165574868_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206215c1f7a55ce4faabf0ab5d165574868_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:20,192 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ca0e3be655dc4d77b36ca17d970f9379, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:20,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ca0e3be655dc4d77b36ca17d970f9379 is 175, key is test_row_0/A:col10/1733465658610/Put/seqid=0 2024-12-06T06:14:20,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741993_1169 (size=39549) 2024-12-06T06:14:20,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465720413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465720413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465720414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465720424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,597 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ca0e3be655dc4d77b36ca17d970f9379 2024-12-06T06:14:20,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d845b290e3464d96a8347d631950ed90 is 50, key is test_row_0/B:col10/1733465658610/Put/seqid=0 2024-12-06T06:14:20,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741994_1170 (size=12001) 2024-12-06T06:14:20,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465720804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,810 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465720924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465720924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465720924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:20,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465720934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:21,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d845b290e3464d96a8347d631950ed90 2024-12-06T06:14:21,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/a9b54ab94d7c49f1a2d387537ffaf171 is 50, key is test_row_0/C:col10/1733465658610/Put/seqid=0 2024-12-06T06:14:21,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741995_1171 (size=12001) 2024-12-06T06:14:21,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/a9b54ab94d7c49f1a2d387537ffaf171 2024-12-06T06:14:21,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ca0e3be655dc4d77b36ca17d970f9379 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ca0e3be655dc4d77b36ca17d970f9379 2024-12-06T06:14:21,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ca0e3be655dc4d77b36ca17d970f9379, entries=200, sequenceid=86, filesize=38.6 K 2024-12-06T06:14:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d845b290e3464d96a8347d631950ed90 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d845b290e3464d96a8347d631950ed90 2024-12-06T06:14:21,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d845b290e3464d96a8347d631950ed90, entries=150, sequenceid=86, filesize=11.7 K 2024-12-06T06:14:21,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/a9b54ab94d7c49f1a2d387537ffaf171 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a9b54ab94d7c49f1a2d387537ffaf171 2024-12-06T06:14:21,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a9b54ab94d7c49f1a2d387537ffaf171, entries=150, sequenceid=86, filesize=11.7 K 2024-12-06T06:14:21,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for efacad63d89edd72fefc481a1ce9f017 in 1295ms, sequenceid=86, compaction requested=true 2024-12-06T06:14:21,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:21,061 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:21,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:21,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:21,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:21,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:21,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:21,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:14:21,061 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:21,062 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:21,062 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:21,062 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:21,062 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/a2d1bd685ec540358cd59ddccf32d280, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa7de25415ce4129b15726d6b9d179f8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9ff6993d1b30492db2c2fc84acbea254, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ca0e3be655dc4d77b36ca17d970f9379] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=137.7 K 2024-12-06T06:14:21,062 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:21,063 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/a2d1bd685ec540358cd59ddccf32d280, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa7de25415ce4129b15726d6b9d179f8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9ff6993d1b30492db2c2fc84acbea254, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ca0e3be655dc4d77b36ca17d970f9379] 2024-12-06T06:14:21,063 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:21,063 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d1bd685ec540358cd59ddccf32d280, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465657027 2024-12-06T06:14:21,063 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:21,063 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:21,063 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/a5f9d45634744f6d8ef29284d227dbc4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d2781b4a56ef4ed193c325383d450d12, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/69ac9f6db4564f7fa25f3317628912d2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d845b290e3464d96a8347d631950ed90] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=46.9 K 2024-12-06T06:14:21,064 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7de25415ce4129b15726d6b9d179f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733465657177 2024-12-06T06:14:21,064 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a5f9d45634744f6d8ef29284d227dbc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465657027 2024-12-06T06:14:21,064 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff6993d1b30492db2c2fc84acbea254, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733465657325 2024-12-06T06:14:21,064 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d2781b4a56ef4ed193c325383d450d12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733465657177 2024-12-06T06:14:21,065 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 69ac9f6db4564f7fa25f3317628912d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733465657325 2024-12-06T06:14:21,065 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0e3be655dc4d77b36ca17d970f9379, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733465658606 2024-12-06T06:14:21,065 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d845b290e3464d96a8347d631950ed90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733465658610 2024-12-06T06:14:21,076 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:21,077 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:21,077 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/bfc96b07e94f435b972e3e497197c574 is 50, key is test_row_0/B:col10/1733465658610/Put/seqid=0 2024-12-06T06:14:21,081 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412068cf3d359b8274bdc9f18cc4860064c1f_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:21,085 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412068cf3d359b8274bdc9f18cc4860064c1f_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:21,085 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412068cf3d359b8274bdc9f18cc4860064c1f_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:21,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741996_1172 (size=12139) 2024-12-06T06:14:21,112 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/bfc96b07e94f435b972e3e497197c574 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/bfc96b07e94f435b972e3e497197c574 2024-12-06T06:14:21,124 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into bfc96b07e94f435b972e3e497197c574(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:21,124 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:21,124 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=12, startTime=1733465661061; duration=0sec 2024-12-06T06:14:21,124 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:21,124 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:21,124 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:21,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741997_1173 (size=4469) 2024-12-06T06:14:21,127 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:21,127 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:21,128 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#153 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:21,129 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/6eded4bb8be542e09150fea962adbab2 is 175, key is test_row_0/A:col10/1733465658610/Put/seqid=0 2024-12-06T06:14:21,129 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:21,131 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d8636115608e4357a5a8bccac9ce779a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b9bcf146afda4cd199f68b28f67e466d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/06c1249c8faa49d0b90f9ad341c58834, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a9b54ab94d7c49f1a2d387537ffaf171] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=46.9 K 2024-12-06T06:14:21,134 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d8636115608e4357a5a8bccac9ce779a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465657027 2024-12-06T06:14:21,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741998_1174 (size=31093) 2024-12-06T06:14:21,138 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b9bcf146afda4cd199f68b28f67e466d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733465657177 2024-12-06T06:14:21,138 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c1249c8faa49d0b90f9ad341c58834, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733465657325 2024-12-06T06:14:21,139 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b54ab94d7c49f1a2d387537ffaf171, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733465658610 2024-12-06T06:14:21,148 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/6eded4bb8be542e09150fea962adbab2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/6eded4bb8be542e09150fea962adbab2 2024-12-06T06:14:21,153 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:21,154 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/39f352ca161246afaf557e2d3ef7ca19 is 50, key is test_row_0/C:col10/1733465658610/Put/seqid=0 2024-12-06T06:14:21,156 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into 6eded4bb8be542e09150fea962adbab2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:21,156 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:21,156 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=12, startTime=1733465661060; duration=0sec 2024-12-06T06:14:21,157 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:21,157 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:21,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741999_1175 (size=12139) 2024-12-06T06:14:21,166 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/39f352ca161246afaf557e2d3ef7ca19 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/39f352ca161246afaf557e2d3ef7ca19 2024-12-06T06:14:21,172 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into 39f352ca161246afaf557e2d3ef7ca19(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:21,172 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:21,173 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=12, startTime=1733465661061; duration=0sec 2024-12-06T06:14:21,173 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:21,173 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T06:14:21,184 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:21,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-06T06:14:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T06:14:21,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:21,188 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:21,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T06:14:21,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-06T06:14:21,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:21,342 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T06:14:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e054128a810d4727849155283ebf7c0a_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465659781/Put/seqid=0 2024-12-06T06:14:21,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742000_1176 (size=12154) 2024-12-06T06:14:21,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,370 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e054128a810d4727849155283ebf7c0a_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e054128a810d4727849155283ebf7c0a_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:21,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fa91538e2e0a4f7b91d8c9beda79cf22, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:21,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fa91538e2e0a4f7b91d8c9beda79cf22 is 175, key is test_row_0/A:col10/1733465659781/Put/seqid=0 2024-12-06T06:14:21,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742001_1177 (size=30955) 2024-12-06T06:14:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T06:14:21,777 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fa91538e2e0a4f7b91d8c9beda79cf22 2024-12-06T06:14:21,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/ecbb8db9496f4d7aa573e09e37ac1187 is 50, key is test_row_0/B:col10/1733465659781/Put/seqid=0 2024-12-06T06:14:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T06:14:21,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742002_1178 (size=12001) 2024-12-06T06:14:21,827 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/ecbb8db9496f4d7aa573e09e37ac1187 2024-12-06T06:14:21,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/e36ef84d17bf463a8d9666633f64e3a3 is 50, key is test_row_0/C:col10/1733465659781/Put/seqid=0 2024-12-06T06:14:21,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742003_1179 (size=12001) 2024-12-06T06:14:21,861 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/e36ef84d17bf463a8d9666633f64e3a3 2024-12-06T06:14:21,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fa91538e2e0a4f7b91d8c9beda79cf22 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa91538e2e0a4f7b91d8c9beda79cf22 2024-12-06T06:14:21,874 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa91538e2e0a4f7b91d8c9beda79cf22, entries=150, sequenceid=101, filesize=30.2 K 2024-12-06T06:14:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/ecbb8db9496f4d7aa573e09e37ac1187 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/ecbb8db9496f4d7aa573e09e37ac1187 2024-12-06T06:14:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,881 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/ecbb8db9496f4d7aa573e09e37ac1187, entries=150, sequenceid=101, filesize=11.7 K 2024-12-06T06:14:21,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/e36ef84d17bf463a8d9666633f64e3a3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e36ef84d17bf463a8d9666633f64e3a3 2024-12-06T06:14:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,886 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e36ef84d17bf463a8d9666633f64e3a3, entries=150, sequenceid=101, filesize=11.7 K 2024-12-06T06:14:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,887 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for efacad63d89edd72fefc481a1ce9f017 in 546ms, sequenceid=101, compaction requested=false 2024-12-06T06:14:21,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:21,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-06T06:14:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-06T06:14:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-06T06:14:21,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 701 msec 2024-12-06T06:14:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 708 msec 2024-12-06T06:14:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:22,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061ec43441427246508a38d8fe022757a4_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465662004/Put/seqid=0 2024-12-06T06:14:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742004_1180 (size=21918) 2024-12-06T06:14:22,052 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733465662031, max timestamp=1733465662050Num uniques 859; 2024-12-06T06:14:22,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-06T06:14:22,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465662031, max timestamp=1733465662050Num uniques 859; 2024-12-06T06:14:22,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465662031, max timestamp=1733465662050Num uniques 859; 2024-12-06T06:14:22,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:22,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:22,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061ec43441427246508a38d8fe022757a4_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061ec43441427246508a38d8fe022757a4_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:22,068 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/98c2f98efeff4ac78c4984e52742304b, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:22,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/98c2f98efeff4ac78c4984e52742304b is 175, key is test_row_0/A:col10/1733465662004/Put/seqid=0 2024-12-06T06:14:22,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742005_1181 (size=65323) 2024-12-06T06:14:22,085 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/98c2f98efeff4ac78c4984e52742304b 2024-12-06T06:14:22,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465722083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465722085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465722090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/85745b1ecea74a149b70823babb9e72d is 50, key is test_row_0/B:col10/1733465662004/Put/seqid=0 2024-12-06T06:14:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465722092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742006_1182 (size=12001) 2024-12-06T06:14:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465722193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465722193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465722204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465722204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T06:14:22,324 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:22,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-06T06:14:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T06:14:22,327 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:22,327 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:22,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:22,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465722404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465722404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465722414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465722414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T06:14:22,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-06T06:14:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/85745b1ecea74a149b70823babb9e72d 2024-12-06T06:14:22,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/a8453a7250a748b9848efc5a7a9d0e3f is 50, key is test_row_0/C:col10/1733465662004/Put/seqid=0 2024-12-06T06:14:22,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742007_1183 (size=12001) 2024-12-06T06:14:22,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-06T06:14:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T06:14:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465722714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465722714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465722734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465722734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-06T06:14:22,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:22,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:22,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465722834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:22,839 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:22,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/a8453a7250a748b9848efc5a7a9d0e3f 2024-12-06T06:14:22,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/98c2f98efeff4ac78c4984e52742304b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/98c2f98efeff4ac78c4984e52742304b 2024-12-06T06:14:22,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/98c2f98efeff4ac78c4984e52742304b, entries=350, sequenceid=111, filesize=63.8 K 2024-12-06T06:14:22,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/85745b1ecea74a149b70823babb9e72d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/85745b1ecea74a149b70823babb9e72d 2024-12-06T06:14:22,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/85745b1ecea74a149b70823babb9e72d, entries=150, sequenceid=111, filesize=11.7 K 2024-12-06T06:14:22,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/a8453a7250a748b9848efc5a7a9d0e3f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a8453a7250a748b9848efc5a7a9d0e3f 2024-12-06T06:14:22,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a8453a7250a748b9848efc5a7a9d0e3f, entries=150, sequenceid=111, filesize=11.7 K 2024-12-06T06:14:22,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for efacad63d89edd72fefc481a1ce9f017 in 935ms, sequenceid=111, compaction requested=true 2024-12-06T06:14:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:22,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-06T06:14:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:22,940 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:22,940 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:22,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,941 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=136.42 KB heapSize=380.69 KB 2024-12-06T06:14:22,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:22,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:22,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:22,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:22,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:22,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:22,943 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:22,943 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:22,943 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:22,943 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:22,943 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,943 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,943 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/6eded4bb8be542e09150fea962adbab2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa91538e2e0a4f7b91d8c9beda79cf22, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/98c2f98efeff4ac78c4984e52742304b] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=124.4 K 2024-12-06T06:14:22,943 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:22,943 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/bfc96b07e94f435b972e3e497197c574, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/ecbb8db9496f4d7aa573e09e37ac1187, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/85745b1ecea74a149b70823babb9e72d] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=35.3 K 2024-12-06T06:14:22,943 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/6eded4bb8be542e09150fea962adbab2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa91538e2e0a4f7b91d8c9beda79cf22, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/98c2f98efeff4ac78c4984e52742304b] 2024-12-06T06:14:22,944 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eded4bb8be542e09150fea962adbab2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733465658610 2024-12-06T06:14:22,944 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc96b07e94f435b972e3e497197c574, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733465658610 2024-12-06T06:14:22,945 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ecbb8db9496f4d7aa573e09e37ac1187, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733465659780 2024-12-06T06:14:22,945 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa91538e2e0a4f7b91d8c9beda79cf22, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733465659780 2024-12-06T06:14:22,946 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c2f98efeff4ac78c4984e52742304b, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733465661961 2024-12-06T06:14:22,946 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 85745b1ecea74a149b70823babb9e72d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733465661961 2024-12-06T06:14:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T06:14:22,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120622b271d2fccf49e3b3585198a04886f0_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465662089/Put/seqid=0 2024-12-06T06:14:22,963 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:22,963 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/c5e8fa12dd2f4edc803b2476cd52d48a is 50, key is test_row_0/B:col10/1733465662004/Put/seqid=0 2024-12-06T06:14:22,965 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:22,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742008_1184 (size=12304) 2024-12-06T06:14:22,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:22,973 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120622b271d2fccf49e3b3585198a04886f0_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120622b271d2fccf49e3b3585198a04886f0_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:22,973 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206755992b048224b6cb9dc078d323a237d_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:22,976 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206755992b048224b6cb9dc078d323a237d_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:22,976 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206755992b048224b6cb9dc078d323a237d_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:22,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fdf162b9edb4481988efda588f358057, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:22,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fdf162b9edb4481988efda588f358057 is 175, key is test_row_0/A:col10/1733465662089/Put/seqid=0 2024-12-06T06:14:22,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742009_1185 (size=12241) 2024-12-06T06:14:22,992 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/c5e8fa12dd2f4edc803b2476cd52d48a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/c5e8fa12dd2f4edc803b2476cd52d48a 2024-12-06T06:14:23,000 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into c5e8fa12dd2f4edc803b2476cd52d48a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:23,000 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:23,000 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=13, startTime=1733465662940; duration=0sec 2024-12-06T06:14:23,000 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:23,000 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:23,000 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:23,001 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:23,001 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:23,001 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:23,001 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/39f352ca161246afaf557e2d3ef7ca19, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e36ef84d17bf463a8d9666633f64e3a3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a8453a7250a748b9848efc5a7a9d0e3f] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=35.3 K 2024-12-06T06:14:23,001 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 39f352ca161246afaf557e2d3ef7ca19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733465658610 2024-12-06T06:14:23,002 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e36ef84d17bf463a8d9666633f64e3a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733465659780 2024-12-06T06:14:23,003 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a8453a7250a748b9848efc5a7a9d0e3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733465661961 2024-12-06T06:14:23,020 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:23,020 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/d9707f7fcb0040dfa3d3e673eefbd5ec is 50, key is test_row_0/C:col10/1733465662004/Put/seqid=0 2024-12-06T06:14:23,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742010_1186 (size=4469) 2024-12-06T06:14:23,035 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#163 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:23,036 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/42f07c5bac314ec4bb80764734bdcb3e is 175, key is test_row_0/A:col10/1733465662004/Put/seqid=0 2024-12-06T06:14:23,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742011_1187 (size=31105) 2024-12-06T06:14:23,046 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fdf162b9edb4481988efda588f358057 2024-12-06T06:14:23,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742012_1188 (size=12241) 2024-12-06T06:14:23,074 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/d9707f7fcb0040dfa3d3e673eefbd5ec as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d9707f7fcb0040dfa3d3e673eefbd5ec 2024-12-06T06:14:23,083 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into d9707f7fcb0040dfa3d3e673eefbd5ec(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:23,083 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:23,083 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=13, startTime=1733465662940; duration=0sec 2024-12-06T06:14:23,083 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:23,087 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:23,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742013_1189 (size=31195) 2024-12-06T06:14:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d651846ac9ec4c7daca796f5c1420c1e is 50, key is test_row_0/B:col10/1733465662089/Put/seqid=0 2024-12-06T06:14:23,108 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/42f07c5bac314ec4bb80764734bdcb3e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/42f07c5bac314ec4bb80764734bdcb3e 2024-12-06T06:14:23,115 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into 42f07c5bac314ec4bb80764734bdcb3e(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:23,115 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:23,115 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=13, startTime=1733465662940; duration=0sec 2024-12-06T06:14:23,116 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:23,116 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:23,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742014_1190 (size=12151) 2024-12-06T06:14:23,132 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d651846ac9ec4c7daca796f5c1420c1e 2024-12-06T06:14:23,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/89df661ccf14479d85771b3e1bdfad7a is 50, key is test_row_0/C:col10/1733465662089/Put/seqid=0 2024-12-06T06:14:23,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742015_1191 (size=12151) 2024-12-06T06:14:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:23,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465723241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465723242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:23,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465723243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465723254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465723354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465723354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T06:14:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465723564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465723564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:23,576 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/89df661ccf14479d85771b3e1bdfad7a 2024-12-06T06:14:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/fdf162b9edb4481988efda588f358057 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fdf162b9edb4481988efda588f358057 2024-12-06T06:14:23,593 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fdf162b9edb4481988efda588f358057, entries=150, sequenceid=146, filesize=30.4 K 2024-12-06T06:14:23,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d651846ac9ec4c7daca796f5c1420c1e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d651846ac9ec4c7daca796f5c1420c1e 2024-12-06T06:14:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,600 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d651846ac9ec4c7daca796f5c1420c1e, entries=150, sequenceid=146, filesize=11.9 K 2024-12-06T06:14:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/89df661ccf14479d85771b3e1bdfad7a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/89df661ccf14479d85771b3e1bdfad7a 2024-12-06T06:14:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,610 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/89df661ccf14479d85771b3e1bdfad7a, entries=150, sequenceid=146, filesize=11.9 K 2024-12-06T06:14:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,612 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~380.64 KB/389776, currentSize=46.96 KB/48090 for efacad63d89edd72fefc481a1ce9f017 in 672ms, sequenceid=146, compaction requested=false 2024-12-06T06:14:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-06T06:14:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-06T06:14:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-06T06:14:23,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2880 sec 2024-12-06T06:14:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2930 sec 2024-12-06T06:14:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T06:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062b0ac4209575429180ccfd3bbbaa398c_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465663239/Put/seqid=0 2024-12-06T06:14:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742016_1192 (size=12304) 2024-12-06T06:14:23,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:23,954 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062b0ac4209575429180ccfd3bbbaa398c_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b0ac4209575429180ccfd3bbbaa398c_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:23,955 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/7647e908e3b347c084fd51048c63391a, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:23,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/7647e908e3b347c084fd51048c63391a is 175, key is test_row_0/A:col10/1733465663239/Put/seqid=0 2024-12-06T06:14:23,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742017_1193 (size=31105) 2024-12-06T06:14:23,977 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/7647e908e3b347c084fd51048c63391a 2024-12-06T06:14:23,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/16312bbdf6434b05bc0ac639863df40f is 50, key is test_row_0/B:col10/1733465663239/Put/seqid=0 2024-12-06T06:14:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465723987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:23,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733465663965, max timestamp=1733465663989Num uniques 859; 2024-12-06T06:14:23,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T06:14:23,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733465663965, max timestamp=1733465663989Num uniques 859; 2024-12-06T06:14:23,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T06:14:23,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733465663965, max timestamp=1733465663989Num uniques 859; 2024-12-06T06:14:23,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-06T06:14:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742018_1194 (size=12151) 2024-12-06T06:14:24,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/16312bbdf6434b05bc0ac639863df40f 2024-12-06T06:14:24,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/5d7f800dd48d43dd8756eae52b50a885 is 50, key is test_row_0/C:col10/1733465663239/Put/seqid=0 2024-12-06T06:14:24,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742019_1195 (size=12151) 2024-12-06T06:14:24,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/5d7f800dd48d43dd8756eae52b50a885 2024-12-06T06:14:24,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/7647e908e3b347c084fd51048c63391a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7647e908e3b347c084fd51048c63391a 2024-12-06T06:14:24,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7647e908e3b347c084fd51048c63391a, entries=150, sequenceid=160, filesize=30.4 K 2024-12-06T06:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/16312bbdf6434b05bc0ac639863df40f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16312bbdf6434b05bc0ac639863df40f 2024-12-06T06:14:24,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16312bbdf6434b05bc0ac639863df40f, entries=150, sequenceid=160, filesize=11.9 K 2024-12-06T06:14:24,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/5d7f800dd48d43dd8756eae52b50a885 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/5d7f800dd48d43dd8756eae52b50a885 2024-12-06T06:14:24,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/5d7f800dd48d43dd8756eae52b50a885, entries=150, sequenceid=160, filesize=11.9 K 2024-12-06T06:14:24,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for efacad63d89edd72fefc481a1ce9f017 in 203ms, sequenceid=160, compaction requested=true 2024-12-06T06:14:24,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:24,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:24,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:24,098 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:24,098 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:24,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:24,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:24,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:24,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:24,099 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:24,099 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:24,099 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,099 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/42f07c5bac314ec4bb80764734bdcb3e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fdf162b9edb4481988efda588f358057, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7647e908e3b347c084fd51048c63391a] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=91.2 K 2024-12-06T06:14:24,099 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,099 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/42f07c5bac314ec4bb80764734bdcb3e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fdf162b9edb4481988efda588f358057, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7647e908e3b347c084fd51048c63391a] 2024-12-06T06:14:24,100 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f07c5bac314ec4bb80764734bdcb3e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733465661961 2024-12-06T06:14:24,100 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdf162b9edb4481988efda588f358057, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733465662077 2024-12-06T06:14:24,100 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7647e908e3b347c084fd51048c63391a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733465663239 2024-12-06T06:14:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:24,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-06T06:14:24,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:24,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:24,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:24,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:24,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:24,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:24,106 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:24,106 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:24,106 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,107 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/c5e8fa12dd2f4edc803b2476cd52d48a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d651846ac9ec4c7daca796f5c1420c1e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16312bbdf6434b05bc0ac639863df40f] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=35.7 K 2024-12-06T06:14:24,107 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting c5e8fa12dd2f4edc803b2476cd52d48a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733465661961 2024-12-06T06:14:24,108 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d651846ac9ec4c7daca796f5c1420c1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733465662077 2024-12-06T06:14:24,109 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 16312bbdf6434b05bc0ac639863df40f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733465663239 2024-12-06T06:14:24,122 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:24,134 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:24,134 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/08a2f375fe9c41818502ad7eaa4b2d8b is 50, key is test_row_0/B:col10/1733465663239/Put/seqid=0 2024-12-06T06:14:24,148 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206d1e8edf7a9df4a04b6b2e8cd1fa23cff_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:24,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465724138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,150 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206d1e8edf7a9df4a04b6b2e8cd1fa23cff_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:24,150 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d1e8edf7a9df4a04b6b2e8cd1fa23cff_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465724142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742020_1196 (size=12493) 2024-12-06T06:14:24,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063fb23419fb3846aabe5956ec5856367e_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465664045/Put/seqid=0 2024-12-06T06:14:24,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742021_1197 (size=4469) 2024-12-06T06:14:24,174 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#170 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:24,175 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/b0a8b4ecd3da4ea3a4613ca9bf7a2941 is 175, key is test_row_0/A:col10/1733465663239/Put/seqid=0 2024-12-06T06:14:24,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742022_1198 (size=17284) 2024-12-06T06:14:24,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742023_1199 (size=31447) 2024-12-06T06:14:24,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465724253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465724254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,255 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465724264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465724274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,279 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465724463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T06:14:24,477 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:24,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:24,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465724474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-06T06:14:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,481 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T06:14:24,482 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:24,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:24,575 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/08a2f375fe9c41818502ad7eaa4b2d8b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/08a2f375fe9c41818502ad7eaa4b2d8b 2024-12-06T06:14:24,581 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into 08a2f375fe9c41818502ad7eaa4b2d8b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:24,581 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:24,581 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=13, startTime=1733465664098; duration=0sec 2024-12-06T06:14:24,581 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:24,582 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:24,582 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:24,583 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:24,583 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:24,583 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,583 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d9707f7fcb0040dfa3d3e673eefbd5ec, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/89df661ccf14479d85771b3e1bdfad7a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/5d7f800dd48d43dd8756eae52b50a885] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=35.7 K 2024-12-06T06:14:24,584 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d9707f7fcb0040dfa3d3e673eefbd5ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733465661961 2024-12-06T06:14:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T06:14:24,584 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 89df661ccf14479d85771b3e1bdfad7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733465662077 2024-12-06T06:14:24,584 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d7f800dd48d43dd8756eae52b50a885, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733465663239 2024-12-06T06:14:24,597 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:24,598 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/1a1e21b3e9524984ab5ab431af022279 is 50, key is test_row_0/C:col10/1733465663239/Put/seqid=0 2024-12-06T06:14:24,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:24,617 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063fb23419fb3846aabe5956ec5856367e_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063fb23419fb3846aabe5956ec5856367e_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:24,618 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/b0a8b4ecd3da4ea3a4613ca9bf7a2941 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b0a8b4ecd3da4ea3a4613ca9bf7a2941 2024-12-06T06:14:24,619 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/1c3f1d931e3d46438d520db13ecd2707, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:24,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/1c3f1d931e3d46438d520db13ecd2707 is 175, key is test_row_0/A:col10/1733465664045/Put/seqid=0 2024-12-06T06:14:24,630 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into b0a8b4ecd3da4ea3a4613ca9bf7a2941(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:24,630 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:24,630 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=13, startTime=1733465664098; duration=0sec 2024-12-06T06:14:24,630 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:24,630 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:24,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742024_1200 (size=12493) 2024-12-06T06:14:24,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T06:14:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:24,658 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/1a1e21b3e9524984ab5ab431af022279 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1a1e21b3e9524984ab5ab431af022279 2024-12-06T06:14:24,665 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into 1a1e21b3e9524984ab5ab431af022279(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:24,665 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:24,665 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=13, startTime=1733465664098; duration=0sec 2024-12-06T06:14:24,665 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:24,665 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:24,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742025_1201 (size=48389) 2024-12-06T06:14:24,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465724774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465724786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:24,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T06:14:24,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T06:14:24,939 INFO [master/6336e8474e1c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-06T06:14:24,939 INFO [master/6336e8474e1c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-06T06:14:24,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T06:14:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:24,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:25,084 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/1c3f1d931e3d46438d520db13ecd2707 2024-12-06T06:14:25,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/cb26b446304a4b23a01833c07daa5669 is 50, key is test_row_0/B:col10/1733465664045/Put/seqid=0 2024-12-06T06:14:25,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T06:14:25,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:25,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:25,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:25,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:25,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T06:14:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:25,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742026_1202 (size=12151) 2024-12-06T06:14:25,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/cb26b446304a4b23a01833c07daa5669 2024-12-06T06:14:25,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/c70d15958f744c3a8556f6c843fbee5d is 50, key is test_row_0/C:col10/1733465664045/Put/seqid=0 2024-12-06T06:14:25,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742027_1203 (size=12151) 2024-12-06T06:14:25,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/c70d15958f744c3a8556f6c843fbee5d 2024-12-06T06:14:25,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/1c3f1d931e3d46438d520db13ecd2707 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/1c3f1d931e3d46438d520db13ecd2707 2024-12-06T06:14:25,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/1c3f1d931e3d46438d520db13ecd2707, entries=250, sequenceid=194, filesize=47.3 K 2024-12-06T06:14:25,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/cb26b446304a4b23a01833c07daa5669 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cb26b446304a4b23a01833c07daa5669 2024-12-06T06:14:25,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cb26b446304a4b23a01833c07daa5669, entries=150, sequenceid=194, filesize=11.9 K 2024-12-06T06:14:25,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/c70d15958f744c3a8556f6c843fbee5d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/c70d15958f744c3a8556f6c843fbee5d 2024-12-06T06:14:25,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/c70d15958f744c3a8556f6c843fbee5d, entries=150, sequenceid=194, filesize=11.9 K 2024-12-06T06:14:25,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for efacad63d89edd72fefc481a1ce9f017 in 1125ms, sequenceid=194, compaction requested=false 2024-12-06T06:14:25,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:25,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T06:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:25,258 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T06:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:25,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c7d085c5608a49aaabdf41df1d3e3dc0_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465664112/Put/seqid=0 2024-12-06T06:14:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:25,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:25,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742028_1204 (size=12304) 2024-12-06T06:14:25,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:25,327 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c7d085c5608a49aaabdf41df1d3e3dc0_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c7d085c5608a49aaabdf41df1d3e3dc0_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:25,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/d468e2b0525140a395d57f08d68c3fc5, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:25,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/d468e2b0525140a395d57f08d68c3fc5 is 175, key is test_row_0/A:col10/1733465664112/Put/seqid=0 2024-12-06T06:14:25,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742029_1205 (size=31105) 2024-12-06T06:14:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465725363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:25,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465665330, max timestamp=1733465665364Num uniques 859; 2024-12-06T06:14:25,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465665330, max timestamp=1733465665364Num uniques 859; 2024-12-06T06:14:25,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:14:25,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:14:25,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465665330, max timestamp=1733465665364Num uniques 859; 2024-12-06T06:14:25,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:14:25,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465725474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465725479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465725594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T06:14:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465725685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:25,764 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/d468e2b0525140a395d57f08d68c3fc5 2024-12-06T06:14:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/07db8f3a276b452c96abacf91801f3ad is 50, key is test_row_0/B:col10/1733465664112/Put/seqid=0 2024-12-06T06:14:25,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742030_1206 (size=12151) 2024-12-06T06:14:25,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465725804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:25,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465725994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465726117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,203 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/07db8f3a276b452c96abacf91801f3ad 2024-12-06T06:14:26,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b8ca3051056c45f5b0e251f532022330 is 50, key is test_row_0/C:col10/1733465664112/Put/seqid=0 2024-12-06T06:14:26,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742031_1207 (size=12151) 2024-12-06T06:14:26,262 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b8ca3051056c45f5b0e251f532022330 2024-12-06T06:14:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/d468e2b0525140a395d57f08d68c3fc5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d468e2b0525140a395d57f08d68c3fc5 2024-12-06T06:14:26,277 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d468e2b0525140a395d57f08d68c3fc5, entries=150, sequenceid=208, filesize=30.4 K 2024-12-06T06:14:26,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/07db8f3a276b452c96abacf91801f3ad as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/07db8f3a276b452c96abacf91801f3ad 2024-12-06T06:14:26,284 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/07db8f3a276b452c96abacf91801f3ad, entries=150, sequenceid=208, filesize=11.9 K 2024-12-06T06:14:26,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b8ca3051056c45f5b0e251f532022330 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b8ca3051056c45f5b0e251f532022330 2024-12-06T06:14:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465726284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,290 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,293 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b8ca3051056c45f5b0e251f532022330, entries=150, sequenceid=208, filesize=11.9 K 2024-12-06T06:14:26,294 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for efacad63d89edd72fefc481a1ce9f017 in 1036ms, sequenceid=208, compaction requested=true 2024-12-06T06:14:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-06T06:14:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-06T06:14:26,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-06T06:14:26,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8140 sec 2024-12-06T06:14:26,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8220 sec 2024-12-06T06:14:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:26,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-06T06:14:26,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:26,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:26,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:26,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:26,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:26,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:26,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206204ff93810de40008e58a434453a654a_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465665474/Put/seqid=0 2024-12-06T06:14:26,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465726348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742032_1208 (size=12304) 2024-12-06T06:14:26,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465726455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465726504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T06:14:26,636 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:26,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-06T06:14:26,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465726628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T06:14:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,640 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:26,641 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:26,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:26,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465726664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T06:14:26,759 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:26,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206204ff93810de40008e58a434453a654a_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206204ff93810de40008e58a434453a654a_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:26,765 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/d43d4fb725f54e33a0fd69dede735629, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:26,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/d43d4fb725f54e33a0fd69dede735629 is 175, key is test_row_0/A:col10/1733465665474/Put/seqid=0 2024-12-06T06:14:26,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742033_1209 (size=31105) 2024-12-06T06:14:26,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:26,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:26,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:26,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465726885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,891 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:26,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:26,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:26,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:26,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:26,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:26,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T06:14:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465726973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:27,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/d43d4fb725f54e33a0fd69dede735629 2024-12-06T06:14:27,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/8c921d59db1546ee99682a3bae5511a0 is 50, key is test_row_0/B:col10/1733465665474/Put/seqid=0 2024-12-06T06:14:27,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742034_1210 (size=12151) 2024-12-06T06:14:27,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T06:14:27,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:27,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465727493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:27,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465727524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:27,525 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:27,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/8c921d59db1546ee99682a3bae5511a0 2024-12-06T06:14:27,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/64254ecb00644e1a91ce6cad166aa1a9 is 50, key is test_row_0/C:col10/1733465665474/Put/seqid=0 2024-12-06T06:14:27,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742035_1211 (size=12151) 2024-12-06T06:14:27,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465727654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:27,655 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:27,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T06:14:27,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:27,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:27,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:28,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:28,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:28,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:28,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:28,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/64254ecb00644e1a91ce6cad166aa1a9 2024-12-06T06:14:28,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/d43d4fb725f54e33a0fd69dede735629 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d43d4fb725f54e33a0fd69dede735629 2024-12-06T06:14:28,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d43d4fb725f54e33a0fd69dede735629, entries=150, sequenceid=243, filesize=30.4 K 2024-12-06T06:14:28,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/8c921d59db1546ee99682a3bae5511a0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/8c921d59db1546ee99682a3bae5511a0 2024-12-06T06:14:28,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/8c921d59db1546ee99682a3bae5511a0, entries=150, sequenceid=243, filesize=11.9 K 2024-12-06T06:14:28,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/64254ecb00644e1a91ce6cad166aa1a9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/64254ecb00644e1a91ce6cad166aa1a9 2024-12-06T06:14:28,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/64254ecb00644e1a91ce6cad166aa1a9, entries=150, sequenceid=243, filesize=11.9 K 2024-12-06T06:14:28,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for efacad63d89edd72fefc481a1ce9f017 in 1754ms, sequenceid=243, compaction requested=true 2024-12-06T06:14:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:28,063 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:28,064 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:28,065 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142046 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:28,065 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:28,065 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:28,065 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b0a8b4ecd3da4ea3a4613ca9bf7a2941, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/1c3f1d931e3d46438d520db13ecd2707, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d468e2b0525140a395d57f08d68c3fc5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d43d4fb725f54e33a0fd69dede735629] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=138.7 K 2024-12-06T06:14:28,065 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:28,065 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b0a8b4ecd3da4ea3a4613ca9bf7a2941, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/1c3f1d931e3d46438d520db13ecd2707, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d468e2b0525140a395d57f08d68c3fc5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d43d4fb725f54e33a0fd69dede735629] 2024-12-06T06:14:28,066 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0a8b4ecd3da4ea3a4613ca9bf7a2941, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733465663239 2024-12-06T06:14:28,066 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:28,066 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:28,066 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:28,066 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/08a2f375fe9c41818502ad7eaa4b2d8b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cb26b446304a4b23a01833c07daa5669, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/07db8f3a276b452c96abacf91801f3ad, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/8c921d59db1546ee99682a3bae5511a0] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=47.8 K 2024-12-06T06:14:28,066 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3f1d931e3d46438d520db13ecd2707, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733465664045 2024-12-06T06:14:28,067 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 08a2f375fe9c41818502ad7eaa4b2d8b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733465663239 2024-12-06T06:14:28,067 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d468e2b0525140a395d57f08d68c3fc5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733465664112 2024-12-06T06:14:28,069 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d43d4fb725f54e33a0fd69dede735629, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733465665424 2024-12-06T06:14:28,069 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting cb26b446304a4b23a01833c07daa5669, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733465664045 2024-12-06T06:14:28,069 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 07db8f3a276b452c96abacf91801f3ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733465664112 2024-12-06T06:14:28,070 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c921d59db1546ee99682a3bae5511a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733465665424 2024-12-06T06:14:28,082 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:28,083 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/cf64754db0f04e748b667208bff05ec9 is 50, key is test_row_0/B:col10/1733465665474/Put/seqid=0 2024-12-06T06:14:28,089 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:28,092 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412062daaf73eec5c4ded88b9c41bd4b9c6df_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:28,095 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412062daaf73eec5c4ded88b9c41bd4b9c6df_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:28,095 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062daaf73eec5c4ded88b9c41bd4b9c6df_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:28,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742036_1212 (size=12629) 2024-12-06T06:14:28,108 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/cf64754db0f04e748b667208bff05ec9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cf64754db0f04e748b667208bff05ec9 2024-12-06T06:14:28,117 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into cf64754db0f04e748b667208bff05ec9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:28,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:28,117 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=12, startTime=1733465668064; duration=0sec 2024-12-06T06:14:28,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:28,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:28,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:14:28,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742037_1213 (size=4469) 2024-12-06T06:14:28,121 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:14:28,121 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#183 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:28,122 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:28,122 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:28,122 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1a1e21b3e9524984ab5ab431af022279, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/c70d15958f744c3a8556f6c843fbee5d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b8ca3051056c45f5b0e251f532022330, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/64254ecb00644e1a91ce6cad166aa1a9] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=47.8 K 2024-12-06T06:14:28,122 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/c64c2d01152b4e0398c576069004ddd9 is 175, key is test_row_0/A:col10/1733465665474/Put/seqid=0 2024-12-06T06:14:28,123 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a1e21b3e9524984ab5ab431af022279, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733465663239 2024-12-06T06:14:28,123 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting c70d15958f744c3a8556f6c843fbee5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733465664045 2024-12-06T06:14:28,125 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ca3051056c45f5b0e251f532022330, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733465664112 2024-12-06T06:14:28,125 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 64254ecb00644e1a91ce6cad166aa1a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733465665424 2024-12-06T06:14:28,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742038_1214 (size=31583) 2024-12-06T06:14:28,147 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/c64c2d01152b4e0398c576069004ddd9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/c64c2d01152b4e0398c576069004ddd9 2024-12-06T06:14:28,155 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:28,156 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b2ea04f690bd4482a85aac23d8cc511c is 50, key is test_row_0/C:col10/1733465665474/Put/seqid=0 2024-12-06T06:14:28,163 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into c64c2d01152b4e0398c576069004ddd9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:28,163 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:28,163 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=12, startTime=1733465668063; duration=0sec 2024-12-06T06:14:28,163 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:28,163 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:28,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742039_1215 (size=12629) 2024-12-06T06:14:28,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-06T06:14:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:28,180 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:14:28,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:28,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:28,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120631e92748efcf4767b15ab97b7c478a92_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465666346/Put/seqid=0 2024-12-06T06:14:28,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742040_1216 (size=12304) 2024-12-06T06:14:28,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:28,221 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120631e92748efcf4767b15ab97b7c478a92_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120631e92748efcf4767b15ab97b7c478a92_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/18736f4b4f6e4560bda796c1ee461a38, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:28,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/18736f4b4f6e4560bda796c1ee461a38 is 175, key is test_row_0/A:col10/1733465666346/Put/seqid=0 2024-12-06T06:14:28,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742041_1217 (size=31105) 2024-12-06T06:14:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:28,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:28,588 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b2ea04f690bd4482a85aac23d8cc511c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b2ea04f690bd4482a85aac23d8cc511c 2024-12-06T06:14:28,595 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into b2ea04f690bd4482a85aac23d8cc511c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:28,595 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:28,595 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=12, startTime=1733465668064; duration=0sec 2024-12-06T06:14:28,596 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:28,596 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:28,640 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/18736f4b4f6e4560bda796c1ee461a38 2024-12-06T06:14:28,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465668618, max timestamp=1733465668643Num uniques 859; 2024-12-06T06:14:28,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:14:28,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465668618, max timestamp=1733465668643Num uniques 859; 2024-12-06T06:14:28,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:14:28,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733465668626, max timestamp=1733465668643Num uniques 859; 2024-12-06T06:14:28,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T06:14:28,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/99e689e5ffb341128e898bd3de0f78d2 is 50, key is test_row_0/B:col10/1733465666346/Put/seqid=0 2024-12-06T06:14:28,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742042_1218 (size=12151) 2024-12-06T06:14:28,673 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/99e689e5ffb341128e898bd3de0f78d2 2024-12-06T06:14:28,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/bdfa597d947a4202a1575ea6a1f89cec is 50, key is test_row_0/C:col10/1733465666346/Put/seqid=0 2024-12-06T06:14:28,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742043_1219 (size=12151) 2024-12-06T06:14:28,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465728749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:28,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:28,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:28,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T06:14:28,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465728864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:29,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465729073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:29,124 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/bdfa597d947a4202a1575ea6a1f89cec 2024-12-06T06:14:29,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/18736f4b4f6e4560bda796c1ee461a38 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/18736f4b4f6e4560bda796c1ee461a38 2024-12-06T06:14:29,149 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/18736f4b4f6e4560bda796c1ee461a38, entries=150, sequenceid=255, filesize=30.4 K 2024-12-06T06:14:29,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/99e689e5ffb341128e898bd3de0f78d2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e689e5ffb341128e898bd3de0f78d2 2024-12-06T06:14:29,156 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e689e5ffb341128e898bd3de0f78d2, entries=150, sequenceid=255, filesize=11.9 K 2024-12-06T06:14:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/bdfa597d947a4202a1575ea6a1f89cec as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/bdfa597d947a4202a1575ea6a1f89cec 2024-12-06T06:14:29,170 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/bdfa597d947a4202a1575ea6a1f89cec, entries=150, sequenceid=255, filesize=11.9 K 2024-12-06T06:14:29,172 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139285 for efacad63d89edd72fefc481a1ce9f017 in 992ms, sequenceid=255, compaction requested=false 2024-12-06T06:14:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-06T06:14:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-06T06:14:29,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-06T06:14:29,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5320 sec 2024-12-06T06:14:29,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.5410 sec 2024-12-06T06:14:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:29,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-12-06T06:14:29,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:29,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:29,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:29,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:29,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:29,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:29,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412067cab84fd41454522ab2436b8e80e6c39_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465669385/Put/seqid=0 2024-12-06T06:14:29,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465729423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:29,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742044_1220 (size=14994) 2024-12-06T06:14:29,455 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:29,460 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412067cab84fd41454522ab2436b8e80e6c39_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067cab84fd41454522ab2436b8e80e6c39_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:29,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/b7c525b9caf14977829997c696d21cd3, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:29,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/b7c525b9caf14977829997c696d21cd3 is 175, key is test_row_0/A:col10/1733465669385/Put/seqid=0 2024-12-06T06:14:29,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742045_1221 (size=39949) 2024-12-06T06:14:29,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/b7c525b9caf14977829997c696d21cd3 2024-12-06T06:14:29,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/117299a7db2e4173a4f66881c37888df is 50, key is test_row_0/B:col10/1733465669385/Put/seqid=0 2024-12-06T06:14:29,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465729546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:29,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465729546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:29,551 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:29,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742046_1222 (size=12301) 2024-12-06T06:14:29,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/117299a7db2e4173a4f66881c37888df 2024-12-06T06:14:29,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/3298ac5f8e064efa961a185ee7c58ef3 is 50, key is test_row_0/C:col10/1733465669385/Put/seqid=0 2024-12-06T06:14:29,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742047_1223 (size=12301) 2024-12-06T06:14:29,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/3298ac5f8e064efa961a185ee7c58ef3 2024-12-06T06:14:29,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/b7c525b9caf14977829997c696d21cd3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b7c525b9caf14977829997c696d21cd3 2024-12-06T06:14:29,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b7c525b9caf14977829997c696d21cd3, entries=200, sequenceid=293, filesize=39.0 K 2024-12-06T06:14:29,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/117299a7db2e4173a4f66881c37888df as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/117299a7db2e4173a4f66881c37888df 2024-12-06T06:14:29,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/117299a7db2e4173a4f66881c37888df, entries=150, sequenceid=293, filesize=12.0 K 2024-12-06T06:14:29,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/3298ac5f8e064efa961a185ee7c58ef3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/3298ac5f8e064efa961a185ee7c58ef3 2024-12-06T06:14:29,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/3298ac5f8e064efa961a185ee7c58ef3, entries=150, sequenceid=293, filesize=12.0 K 2024-12-06T06:14:29,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~142.73 KB/146155, heapSize ~404.34 KB/414040, currentSize=40.25 KB/41220 for efacad63d89edd72fefc481a1ce9f017 in 247ms, sequenceid=293, compaction requested=true 2024-12-06T06:14:29,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:29,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:29,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:29,634 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:29,634 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:29,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:29,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:29,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:29,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:29,635 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:29,635 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:29,635 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:29,635 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:29,635 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:29,635 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:29,635 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/c64c2d01152b4e0398c576069004ddd9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/18736f4b4f6e4560bda796c1ee461a38, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b7c525b9caf14977829997c696d21cd3] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=100.2 K 2024-12-06T06:14:29,635 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cf64754db0f04e748b667208bff05ec9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e689e5ffb341128e898bd3de0f78d2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/117299a7db2e4173a4f66881c37888df] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.2 K 2024-12-06T06:14:29,635 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:29,636 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/c64c2d01152b4e0398c576069004ddd9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/18736f4b4f6e4560bda796c1ee461a38, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b7c525b9caf14977829997c696d21cd3] 2024-12-06T06:14:29,637 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c64c2d01152b4e0398c576069004ddd9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733465665424 2024-12-06T06:14:29,637 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting cf64754db0f04e748b667208bff05ec9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733465665424 2024-12-06T06:14:29,637 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e689e5ffb341128e898bd3de0f78d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733465666325 2024-12-06T06:14:29,638 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18736f4b4f6e4560bda796c1ee461a38, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733465666325 2024-12-06T06:14:29,638 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 117299a7db2e4173a4f66881c37888df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733465668690 2024-12-06T06:14:29,638 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c525b9caf14977829997c696d21cd3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733465668690 2024-12-06T06:14:29,652 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:29,653 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/9b94f5f7c2154dbabda13a813b40be67 is 50, key is test_row_0/B:col10/1733465669385/Put/seqid=0 2024-12-06T06:14:29,654 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:29,664 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412064c129cbe44514e629f1e9ef4646a0625_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:29,670 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412064c129cbe44514e629f1e9ef4646a0625_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:29,671 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064c129cbe44514e629f1e9ef4646a0625_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:29,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:14:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:29,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742048_1224 (size=12881) 2024-12-06T06:14:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:29,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742049_1225 (size=4469) 2024-12-06T06:14:29,710 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#192 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:29,711 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/bec32f0d6375484fa92950fc60593f4e is 175, key is test_row_0/A:col10/1733465669385/Put/seqid=0 2024-12-06T06:14:29,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b1ef4ed4151d4013961e629021d2a807_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465669400/Put/seqid=0 2024-12-06T06:14:29,718 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/9b94f5f7c2154dbabda13a813b40be67 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/9b94f5f7c2154dbabda13a813b40be67 2024-12-06T06:14:29,727 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into 9b94f5f7c2154dbabda13a813b40be67(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:29,727 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:29,727 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=13, startTime=1733465669634; duration=0sec 2024-12-06T06:14:29,727 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:29,727 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:29,727 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:29,730 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:29,730 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:29,730 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:29,730 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b2ea04f690bd4482a85aac23d8cc511c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/bdfa597d947a4202a1575ea6a1f89cec, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/3298ac5f8e064efa961a185ee7c58ef3] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.2 K 2024-12-06T06:14:29,730 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ea04f690bd4482a85aac23d8cc511c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733465665424 2024-12-06T06:14:29,731 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfa597d947a4202a1575ea6a1f89cec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733465666325 2024-12-06T06:14:29,731 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3298ac5f8e064efa961a185ee7c58ef3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733465668690 2024-12-06T06:14:29,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742051_1227 (size=14994) 2024-12-06T06:14:29,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742050_1226 (size=31835) 2024-12-06T06:14:29,789 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:29,790 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/a2e6fa1c515944229e1fa116716e9614 is 50, key is test_row_0/C:col10/1733465669385/Put/seqid=0 2024-12-06T06:14:29,799 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/bec32f0d6375484fa92950fc60593f4e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/bec32f0d6375484fa92950fc60593f4e 2024-12-06T06:14:29,810 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into bec32f0d6375484fa92950fc60593f4e(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:29,811 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:29,811 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=13, startTime=1733465669634; duration=0sec 2024-12-06T06:14:29,811 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:29,811 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:29,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742052_1228 (size=12881) 2024-12-06T06:14:29,835 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/a2e6fa1c515944229e1fa116716e9614 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a2e6fa1c515944229e1fa116716e9614 2024-12-06T06:14:29,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465669810, max timestamp=1733465669834Num uniques 859; 2024-12-06T06:14:29,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:29,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733465669810, max timestamp=1733465669834Num uniques 859; 2024-12-06T06:14:29,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T06:14:29,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733465669810, max timestamp=1733465669834Num uniques 859; 2024-12-06T06:14:29,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-06T06:14:29,844 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into a2e6fa1c515944229e1fa116716e9614(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:29,844 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:29,844 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=13, startTime=1733465669635; duration=0sec 2024-12-06T06:14:29,844 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:29,844 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:29,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465729906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:29,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465729909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465730024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465730024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:30,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b1ef4ed4151d4013961e629021d2a807_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b1ef4ed4151d4013961e629021d2a807_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:30,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/cc4174d44cce4271946a7d1fb130cfde, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:30,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/cc4174d44cce4271946a7d1fb130cfde is 175, key is test_row_0/A:col10/1733465669400/Put/seqid=0 2024-12-06T06:14:30,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742053_1229 (size=39949) 2024-12-06T06:14:30,178 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/cc4174d44cce4271946a7d1fb130cfde 2024-12-06T06:14:30,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/3f4946ae752a426eab256beb39948ea6 is 50, key is test_row_0/B:col10/1733465669400/Put/seqid=0 2024-12-06T06:14:30,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742054_1230 (size=12301) 2024-12-06T06:14:30,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/3f4946ae752a426eab256beb39948ea6 2024-12-06T06:14:30,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/162b35929a374bcca94f650525c71f5d is 50, key is test_row_0/C:col10/1733465669400/Put/seqid=0 2024-12-06T06:14:30,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742055_1231 (size=12301) 2024-12-06T06:14:30,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/162b35929a374bcca94f650525c71f5d 2024-12-06T06:14:30,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/cc4174d44cce4271946a7d1fb130cfde as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cc4174d44cce4271946a7d1fb130cfde 2024-12-06T06:14:30,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465730234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465730234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cc4174d44cce4271946a7d1fb130cfde, entries=200, sequenceid=303, filesize=39.0 K 2024-12-06T06:14:30,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/3f4946ae752a426eab256beb39948ea6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/3f4946ae752a426eab256beb39948ea6 2024-12-06T06:14:30,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/3f4946ae752a426eab256beb39948ea6, entries=150, sequenceid=303, filesize=12.0 K 2024-12-06T06:14:30,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/162b35929a374bcca94f650525c71f5d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/162b35929a374bcca94f650525c71f5d 2024-12-06T06:14:30,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/162b35929a374bcca94f650525c71f5d, entries=150, sequenceid=303, filesize=12.0 K 2024-12-06T06:14:30,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.63 KB/138880 for efacad63d89edd72fefc481a1ce9f017 in 562ms, sequenceid=303, compaction requested=false 2024-12-06T06:14:30,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:30,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=142.33 KB heapSize=408.86 KB 2024-12-06T06:14:30,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:30,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:14:30,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:30,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:14:30,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:30,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:14:30,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412060184b5f03cf24162bdc590c2b4441748_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465670316/Put/seqid=0 2024-12-06T06:14:30,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742056_1232 (size=14994) 2024-12-06T06:14:30,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465730342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465730454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465730543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465730544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465730664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:30,731 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:30,736 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412060184b5f03cf24162bdc590c2b4441748_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060184b5f03cf24162bdc590c2b4441748_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:30,737 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ad7fdac278fe4bb883c3efc01679cfe6, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:30,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ad7fdac278fe4bb883c3efc01679cfe6 is 175, key is test_row_0/A:col10/1733465670316/Put/seqid=0 2024-12-06T06:14:30,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742057_1233 (size=39949) 2024-12-06T06:14:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T06:14:30,794 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:30,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-06T06:14:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T06:14:30,797 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:30,797 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:30,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T06:14:30,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-06T06:14:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:30,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:30,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465730973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465731053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465731053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:31,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-06T06:14:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T06:14:31,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ad7fdac278fe4bb883c3efc01679cfe6 2024-12-06T06:14:31,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/b0ce051dd3924dd0aa4049ab6599b2e6 is 50, key is test_row_0/B:col10/1733465670316/Put/seqid=0 2024-12-06T06:14:31,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742058_1234 (size=12301) 2024-12-06T06:14:31,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/b0ce051dd3924dd0aa4049ab6599b2e6 2024-12-06T06:14:31,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/66efe8bb7c954c9c8bc4d227f44c2420 is 50, key is test_row_0/C:col10/1733465670316/Put/seqid=0 2024-12-06T06:14:31,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742059_1235 (size=12301) 2024-12-06T06:14:31,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/66efe8bb7c954c9c8bc4d227f44c2420 2024-12-06T06:14:31,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ad7fdac278fe4bb883c3efc01679cfe6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ad7fdac278fe4bb883c3efc01679cfe6 2024-12-06T06:14:31,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ad7fdac278fe4bb883c3efc01679cfe6, entries=200, sequenceid=344, filesize=39.0 K 2024-12-06T06:14:31,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/b0ce051dd3924dd0aa4049ab6599b2e6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/b0ce051dd3924dd0aa4049ab6599b2e6 2024-12-06T06:14:31,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/b0ce051dd3924dd0aa4049ab6599b2e6, entries=150, sequenceid=344, filesize=12.0 K 2024-12-06T06:14:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/66efe8bb7c954c9c8bc4d227f44c2420 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/66efe8bb7c954c9c8bc4d227f44c2420 2024-12-06T06:14:31,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-06T06:14:31,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:31,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/66efe8bb7c954c9c8bc4d227f44c2420, entries=150, sequenceid=344, filesize=12.0 K 2024-12-06T06:14:31,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~142.33 KB/145750, heapSize ~408.81 KB/418624, currentSize=40.25 KB/41220 for efacad63d89edd72fefc481a1ce9f017 in 952ms, sequenceid=344, compaction requested=true 2024-12-06T06:14:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:31,270 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:31,270 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:31,271 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:31,271 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:31,271 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,272 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/bec32f0d6375484fa92950fc60593f4e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cc4174d44cce4271946a7d1fb130cfde, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ad7fdac278fe4bb883c3efc01679cfe6] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=109.1 K 2024-12-06T06:14:31,272 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,272 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/bec32f0d6375484fa92950fc60593f4e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cc4174d44cce4271946a7d1fb130cfde, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ad7fdac278fe4bb883c3efc01679cfe6] 2024-12-06T06:14:31,272 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:31,272 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:31,272 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,272 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/9b94f5f7c2154dbabda13a813b40be67, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/3f4946ae752a426eab256beb39948ea6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/b0ce051dd3924dd0aa4049ab6599b2e6] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.6 K 2024-12-06T06:14:31,275 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec32f0d6375484fa92950fc60593f4e, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733465668690 2024-12-06T06:14:31,275 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b94f5f7c2154dbabda13a813b40be67, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733465668690 2024-12-06T06:14:31,275 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc4174d44cce4271946a7d1fb130cfde, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733465669400 2024-12-06T06:14:31,275 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4946ae752a426eab256beb39948ea6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733465669400 2024-12-06T06:14:31,275 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7fdac278fe4bb883c3efc01679cfe6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733465669877 2024-12-06T06:14:31,276 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ce051dd3924dd0aa4049ab6599b2e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733465669877 2024-12-06T06:14:31,285 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:31,296 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:31,297 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/5f6767a8490c4ac38268766fe4e48fb0 is 50, key is test_row_0/B:col10/1733465670316/Put/seqid=0 2024-12-06T06:14:31,309 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206efdedac0231d49acb0790145eb99dddf_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:31,311 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206efdedac0231d49acb0790145eb99dddf_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:31,311 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206efdedac0231d49acb0790145eb99dddf_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:31,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742060_1236 (size=12983) 2024-12-06T06:14:31,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742061_1237 (size=4469) 2024-12-06T06:14:31,341 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#200 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:31,342 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/7ad019894ef949c8a8db549223fbe374 is 175, key is test_row_0/A:col10/1733465670316/Put/seqid=0 2024-12-06T06:14:31,344 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/5f6767a8490c4ac38268766fe4e48fb0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5f6767a8490c4ac38268766fe4e48fb0 2024-12-06T06:14:31,354 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into 5f6767a8490c4ac38268766fe4e48fb0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:31,354 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:31,354 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=13, startTime=1733465671270; duration=0sec 2024-12-06T06:14:31,354 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:31,354 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:31,356 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:31,358 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:31,358 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:31,359 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,359 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a2e6fa1c515944229e1fa116716e9614, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/162b35929a374bcca94f650525c71f5d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/66efe8bb7c954c9c8bc4d227f44c2420] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.6 K 2024-12-06T06:14:31,359 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e6fa1c515944229e1fa116716e9614, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733465668690 2024-12-06T06:14:31,360 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 162b35929a374bcca94f650525c71f5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733465669400 2024-12-06T06:14:31,360 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 66efe8bb7c954c9c8bc4d227f44c2420, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733465669877 2024-12-06T06:14:31,380 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:31,381 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/f63e4678866a42ceb12f8520d8a6c1b5 is 50, key is test_row_0/C:col10/1733465670316/Put/seqid=0 2024-12-06T06:14:31,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742062_1238 (size=31937) 2024-12-06T06:14:31,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742063_1239 (size=12983) 2024-12-06T06:14:31,410 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/f63e4678866a42ceb12f8520d8a6c1b5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f63e4678866a42ceb12f8520d8a6c1b5 2024-12-06T06:14:31,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-06T06:14:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:31,414 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-06T06:14:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:31,418 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into f63e4678866a42ceb12f8520d8a6c1b5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:31,418 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:31,418 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=13, startTime=1733465671270; duration=0sec 2024-12-06T06:14:31,418 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:31,418 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T06:14:31,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206947a4b2122814c558faddd54557ba3a5_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465670338/Put/seqid=0 2024-12-06T06:14:31,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742064_1240 (size=12454) 2024-12-06T06:14:31,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:31,469 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206947a4b2122814c558faddd54557ba3a5_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206947a4b2122814c558faddd54557ba3a5_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:31,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/174b07a60c4943929c769d93b8a45075, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/174b07a60c4943929c769d93b8a45075 is 175, key is test_row_0/A:col10/1733465670338/Put/seqid=0 2024-12-06T06:14:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:31,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:31,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742065_1241 (size=31255) 2024-12-06T06:14:31,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465671545, max timestamp=1733465671573Num uniques 859; 2024-12-06T06:14:31,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:31,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465671545, max timestamp=1733465671573Num uniques 859; 2024-12-06T06:14:31,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:31,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465731634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:31,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465731744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:31,794 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/7ad019894ef949c8a8db549223fbe374 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7ad019894ef949c8a8db549223fbe374 2024-12-06T06:14:31,800 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into 7ad019894ef949c8a8db549223fbe374(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:31,800 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:31,800 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=13, startTime=1733465671270; duration=0sec 2024-12-06T06:14:31,800 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:31,800 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:31,888 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/174b07a60c4943929c769d93b8a45075 2024-12-06T06:14:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/e0310be5312c4e4ab8cff8953d2f0540 is 50, key is test_row_0/B:col10/1733465670338/Put/seqid=0 2024-12-06T06:14:31,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742066_1242 (size=12301) 2024-12-06T06:14:31,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T06:14:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465731953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:32,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465732064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:32,065 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:32,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465732066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:32,067 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:32,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465732263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:32,306 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/e0310be5312c4e4ab8cff8953d2f0540 2024-12-06T06:14:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/f03f07ec01f44ec2a62fa39eb6a9a605 is 50, key is test_row_0/C:col10/1733465670338/Put/seqid=0 2024-12-06T06:14:32,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742067_1243 (size=12301) 2024-12-06T06:14:32,323 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/f03f07ec01f44ec2a62fa39eb6a9a605 2024-12-06T06:14:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/174b07a60c4943929c769d93b8a45075 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/174b07a60c4943929c769d93b8a45075 2024-12-06T06:14:32,333 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/174b07a60c4943929c769d93b8a45075, entries=150, sequenceid=354, filesize=30.5 K 2024-12-06T06:14:32,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/e0310be5312c4e4ab8cff8953d2f0540 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/e0310be5312c4e4ab8cff8953d2f0540 2024-12-06T06:14:32,338 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/e0310be5312c4e4ab8cff8953d2f0540, entries=150, sequenceid=354, filesize=12.0 K 2024-12-06T06:14:32,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/f03f07ec01f44ec2a62fa39eb6a9a605 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f03f07ec01f44ec2a62fa39eb6a9a605 2024-12-06T06:14:32,344 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f03f07ec01f44ec2a62fa39eb6a9a605, entries=150, sequenceid=354, filesize=12.0 K 2024-12-06T06:14:32,345 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=136.42 KB/139690 for efacad63d89edd72fefc481a1ce9f017 in 931ms, sequenceid=354, compaction requested=false 2024-12-06T06:14:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-06T06:14:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-06T06:14:32,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-06T06:14:32,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-12-06T06:14:32,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5560 sec 2024-12-06T06:14:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:32,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=143.13 KB heapSize=413.39 KB 2024-12-06T06:14:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:32,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206287f48cc1bfc4c8fba60ca908fa9be90_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465672774/Put/seqid=0 2024-12-06T06:14:32,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465732799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:32,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742068_1244 (size=14994) 2024-12-06T06:14:32,810 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:32,814 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206287f48cc1bfc4c8fba60ca908fa9be90_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206287f48cc1bfc4c8fba60ca908fa9be90_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:32,815 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/3e19d7be698f467ea5098259e0bc2cd1, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:32,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/3e19d7be698f467ea5098259e0bc2cd1 is 175, key is test_row_0/A:col10/1733465672774/Put/seqid=0 2024-12-06T06:14:32,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742069_1245 (size=39949) 2024-12-06T06:14:32,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465732914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-06T06:14:32,954 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:32,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-06T06:14:32,957 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T06:14:32,958 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:32,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T06:14:33,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-06T06:14:33,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:33,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:33,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:33,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465733124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:33,225 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/3e19d7be698f467ea5098259e0bc2cd1 2024-12-06T06:14:33,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/956e4abe84404dd7908fd11ac13c5f29 is 50, key is test_row_0/B:col10/1733465672774/Put/seqid=0 2024-12-06T06:14:33,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-06T06:14:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T06:14:33,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742070_1246 (size=12301) 2024-12-06T06:14:33,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/956e4abe84404dd7908fd11ac13c5f29 2024-12-06T06:14:33,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/29ac70db79af49ee8e6dde9e3eb1b3e2 is 50, key is test_row_0/C:col10/1733465672774/Put/seqid=0 2024-12-06T06:14:33,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742071_1247 (size=12301) 2024-12-06T06:14:33,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/29ac70db79af49ee8e6dde9e3eb1b3e2 2024-12-06T06:14:33,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/3e19d7be698f467ea5098259e0bc2cd1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/3e19d7be698f467ea5098259e0bc2cd1 2024-12-06T06:14:33,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/3e19d7be698f467ea5098259e0bc2cd1, entries=200, sequenceid=389, filesize=39.0 K 2024-12-06T06:14:33,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/956e4abe84404dd7908fd11ac13c5f29 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/956e4abe84404dd7908fd11ac13c5f29 2024-12-06T06:14:33,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/956e4abe84404dd7908fd11ac13c5f29, entries=150, sequenceid=389, filesize=12.0 K 2024-12-06T06:14:33,371 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-06T06:14:33,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/29ac70db79af49ee8e6dde9e3eb1b3e2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/29ac70db79af49ee8e6dde9e3eb1b3e2 2024-12-06T06:14:33,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/29ac70db79af49ee8e6dde9e3eb1b3e2, entries=150, sequenceid=389, filesize=12.0 K 2024-12-06T06:14:33,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~413.34 KB/423264, currentSize=33.54 KB/34350 for efacad63d89edd72fefc481a1ce9f017 in 608ms, sequenceid=389, compaction requested=true 2024-12-06T06:14:33,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:33,384 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:33,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:33,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:33,385 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:33,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:33,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:33,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:33,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:33,386 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:33,386 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:33,386 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,386 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7ad019894ef949c8a8db549223fbe374, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/174b07a60c4943929c769d93b8a45075, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/3e19d7be698f467ea5098259e0bc2cd1] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=100.7 K 2024-12-06T06:14:33,386 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,386 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7ad019894ef949c8a8db549223fbe374, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/174b07a60c4943929c769d93b8a45075, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/3e19d7be698f467ea5098259e0bc2cd1] 2024-12-06T06:14:33,386 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:33,387 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad019894ef949c8a8db549223fbe374, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733465669877 2024-12-06T06:14:33,387 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:33,387 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,387 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5f6767a8490c4ac38268766fe4e48fb0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/e0310be5312c4e4ab8cff8953d2f0540, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/956e4abe84404dd7908fd11ac13c5f29] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.7 K 2024-12-06T06:14:33,387 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 174b07a60c4943929c769d93b8a45075, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733465670334 2024-12-06T06:14:33,387 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6767a8490c4ac38268766fe4e48fb0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733465669877 2024-12-06T06:14:33,388 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e19d7be698f467ea5098259e0bc2cd1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465671612 2024-12-06T06:14:33,388 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e0310be5312c4e4ab8cff8953d2f0540, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733465670334 2024-12-06T06:14:33,389 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 956e4abe84404dd7908fd11ac13c5f29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465671626 2024-12-06T06:14:33,407 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:33,409 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:33,409 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/973689f9467f4344a07a18dbab66580f is 50, key is test_row_0/B:col10/1733465672774/Put/seqid=0 2024-12-06T06:14:33,417 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206e3c68de9f5364e9784996ccb2ca77a0b_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:33,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-06T06:14:33,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,419 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T06:14:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:33,419 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206e3c68de9f5364e9784996ccb2ca77a0b_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:33,419 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e3c68de9f5364e9784996ccb2ca77a0b_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:33,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:33,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742072_1248 (size=13085) 2024-12-06T06:14:33,447 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/973689f9467f4344a07a18dbab66580f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/973689f9467f4344a07a18dbab66580f 2024-12-06T06:14:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120617a30a05674d462a939203bd6895a092_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465672797/Put/seqid=0 2024-12-06T06:14:33,458 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into 973689f9467f4344a07a18dbab66580f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:33,458 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:33,458 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=13, startTime=1733465673384; duration=0sec 2024-12-06T06:14:33,459 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:33,459 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:33,459 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:33,461 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:33,461 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:33,461 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:33,461 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f63e4678866a42ceb12f8520d8a6c1b5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f03f07ec01f44ec2a62fa39eb6a9a605, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/29ac70db79af49ee8e6dde9e3eb1b3e2] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.7 K 2024-12-06T06:14:33,462 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting f63e4678866a42ceb12f8520d8a6c1b5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733465669877 2024-12-06T06:14:33,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742073_1249 (size=4469) 2024-12-06T06:14:33,463 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting f03f07ec01f44ec2a62fa39eb6a9a605, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733465670334 2024-12-06T06:14:33,464 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ac70db79af49ee8e6dde9e3eb1b3e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465671626 2024-12-06T06:14:33,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742074_1250 (size=12454) 2024-12-06T06:14:33,483 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:33,483 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/8d2f65b264d64ff882e06e02b80477c3 is 50, key is test_row_0/C:col10/1733465672774/Put/seqid=0 2024-12-06T06:14:33,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742075_1251 (size=13085) 2024-12-06T06:14:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465673476, max timestamp=1733465673532Num uniques 859; 2024-12-06T06:14:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465673476, max timestamp=1733465673532Num uniques 859; 2024-12-06T06:14:33,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:33,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465673476, max timestamp=1733465673532Num uniques 859; 2024-12-06T06:14:33,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T06:14:33,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733465673609, max timestamp=1733465673646Num uniques 1068; 2024-12-06T06:14:33,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-06T06:14:33,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733465673611, max timestamp=1733465673646Num uniques 1068; 2024-12-06T06:14:33,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733465673611, max timestamp=1733465673646Num uniques 1068; 2024-12-06T06:14:33,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-06T06:14:33,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-06T06:14:33,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733465673709, max timestamp=1733465673733Num uniques 1177; 2024-12-06T06:14:33,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733465673709, max timestamp=1733465673733Num uniques 1077; 2024-12-06T06:14:33,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733465673709, max timestamp=1733465673733Num uniques 1077; 2024-12-06T06:14:33,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-06T06:14:33,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-06T06:14:33,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-06T06:14:33,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465733796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:33,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465733801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:33,863 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#210 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:33,864 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/9bd87c0e39fb4b4da6945752fbf3eba7 is 175, key is test_row_0/A:col10/1733465672774/Put/seqid=0 2024-12-06T06:14:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742076_1252 (size=32039) 2024-12-06T06:14:33,875 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/9bd87c0e39fb4b4da6945752fbf3eba7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9bd87c0e39fb4b4da6945752fbf3eba7 2024-12-06T06:14:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:33,885 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into 9bd87c0e39fb4b4da6945752fbf3eba7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:33,885 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:33,885 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=13, startTime=1733465673384; duration=0sec 2024-12-06T06:14:33,885 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:33,885 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:33,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120617a30a05674d462a939203bd6895a092_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120617a30a05674d462a939203bd6895a092_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/e71c8b0082864a3eb6da883311fa029c, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:33,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/e71c8b0082864a3eb6da883311fa029c is 175, key is test_row_0/A:col10/1733465672797/Put/seqid=0 2024-12-06T06:14:33,903 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/8d2f65b264d64ff882e06e02b80477c3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/8d2f65b264d64ff882e06e02b80477c3 2024-12-06T06:14:33,909 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into 8d2f65b264d64ff882e06e02b80477c3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:33,909 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:33,909 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=13, startTime=1733465673385; duration=0sec 2024-12-06T06:14:33,910 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:33,910 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:33,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465733904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:33,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465733904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:33,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742077_1253 (size=31255) 2024-12-06T06:14:33,927 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/e71c8b0082864a3eb6da883311fa029c 2024-12-06T06:14:33,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/f260bfdf48e24aa981764bf1ff19a32c is 50, key is test_row_0/B:col10/1733465672797/Put/seqid=0 2024-12-06T06:14:33,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742078_1254 (size=12301) 2024-12-06T06:14:33,955 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/f260bfdf48e24aa981764bf1ff19a32c 2024-12-06T06:14:33,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/1d9b7710b03e4d578fb677320132bba7 is 50, key is test_row_0/C:col10/1733465672797/Put/seqid=0 2024-12-06T06:14:33,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742079_1255 (size=12301) 2024-12-06T06:14:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53572 deadline: 1733465734074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,077 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53564 deadline: 1733465734084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,089 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T06:14:34,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465734113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465734124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,389 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/1d9b7710b03e4d578fb677320132bba7 2024-12-06T06:14:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/e71c8b0082864a3eb6da883311fa029c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/e71c8b0082864a3eb6da883311fa029c 2024-12-06T06:14:34,401 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/e71c8b0082864a3eb6da883311fa029c, entries=150, sequenceid=397, filesize=30.5 K 2024-12-06T06:14:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/f260bfdf48e24aa981764bf1ff19a32c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/f260bfdf48e24aa981764bf1ff19a32c 2024-12-06T06:14:34,406 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/f260bfdf48e24aa981764bf1ff19a32c, entries=150, sequenceid=397, filesize=12.0 K 2024-12-06T06:14:34,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/1d9b7710b03e4d578fb677320132bba7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1d9b7710b03e4d578fb677320132bba7 2024-12-06T06:14:34,411 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1d9b7710b03e4d578fb677320132bba7, entries=150, sequenceid=397, filesize=12.0 K 2024-12-06T06:14:34,411 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for efacad63d89edd72fefc481a1ce9f017 in 993ms, sequenceid=397, compaction requested=false 2024-12-06T06:14:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-06T06:14:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-06T06:14:34,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-06T06:14:34,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4580 sec 2024-12-06T06:14:34,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.4670 sec 2024-12-06T06:14:34,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-12-06T06:14:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:14:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:14:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:14:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:34,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120655330a8bd9d24ba595a1c4f11aeec735_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465673794/Put/seqid=0 2024-12-06T06:14:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465734452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465734453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742080_1256 (size=14994) 2024-12-06T06:14:34,466 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:34,473 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120655330a8bd9d24ba595a1c4f11aeec735_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120655330a8bd9d24ba595a1c4f11aeec735_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:34,477 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/cb92177835a04348a542bd6f7bcbc35e, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:34,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/cb92177835a04348a542bd6f7bcbc35e is 175, key is test_row_0/A:col10/1733465673794/Put/seqid=0 2024-12-06T06:14:34,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742081_1257 (size=39949) 2024-12-06T06:14:34,495 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/cb92177835a04348a542bd6f7bcbc35e 2024-12-06T06:14:34,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/16fa05b2afd74719918e28234bd688b7 is 50, key is test_row_0/B:col10/1733465673794/Put/seqid=0 2024-12-06T06:14:34,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742082_1258 (size=12301) 2024-12-06T06:14:34,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465734564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465734565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465734784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465734786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:34,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/16fa05b2afd74719918e28234bd688b7 2024-12-06T06:14:34,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/378c8961afad4bf08c8f53234e5cc1bf is 50, key is test_row_0/C:col10/1733465673794/Put/seqid=0 2024-12-06T06:14:34,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742083_1259 (size=12301) 2024-12-06T06:14:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T06:14:35,104 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:35,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:35,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465735094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:35,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465735094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-06T06:14:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-06T06:14:35,108 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:35,109 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:35,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-06T06:14:35,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-06T06:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:35,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:35,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/378c8961afad4bf08c8f53234e5cc1bf 2024-12-06T06:14:35,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/cb92177835a04348a542bd6f7bcbc35e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cb92177835a04348a542bd6f7bcbc35e 2024-12-06T06:14:35,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cb92177835a04348a542bd6f7bcbc35e, entries=200, sequenceid=472, filesize=39.0 K 2024-12-06T06:14:35,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/16fa05b2afd74719918e28234bd688b7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16fa05b2afd74719918e28234bd688b7 2024-12-06T06:14:35,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16fa05b2afd74719918e28234bd688b7, entries=150, sequenceid=472, filesize=12.0 K 2024-12-06T06:14:35,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/378c8961afad4bf08c8f53234e5cc1bf as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/378c8961afad4bf08c8f53234e5cc1bf 2024-12-06T06:14:35,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/378c8961afad4bf08c8f53234e5cc1bf, entries=150, sequenceid=472, filesize=12.0 K 2024-12-06T06:14:35,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~434.34 KB/444768, currentSize=26.84 KB/27480 for efacad63d89edd72fefc481a1ce9f017 in 973ms, sequenceid=472, compaction requested=true 2024-12-06T06:14:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:35,406 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:35,406 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:35,408 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:35,408 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:35,408 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:35,408 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9bd87c0e39fb4b4da6945752fbf3eba7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/e71c8b0082864a3eb6da883311fa029c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cb92177835a04348a542bd6f7bcbc35e] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=100.8 K 2024-12-06T06:14:35,408 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:35,408 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9bd87c0e39fb4b4da6945752fbf3eba7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/e71c8b0082864a3eb6da883311fa029c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cb92177835a04348a542bd6f7bcbc35e] 2024-12-06T06:14:35,409 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:35,409 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:35,409 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:35,409 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/973689f9467f4344a07a18dbab66580f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/f260bfdf48e24aa981764bf1ff19a32c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16fa05b2afd74719918e28234bd688b7] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.8 K 2024-12-06T06:14:35,410 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd87c0e39fb4b4da6945752fbf3eba7, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465671626 2024-12-06T06:14:35,410 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 973689f9467f4344a07a18dbab66580f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465671626 2024-12-06T06:14:35,411 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting f260bfdf48e24aa981764bf1ff19a32c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733465672781 2024-12-06T06:14:35,411 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71c8b0082864a3eb6da883311fa029c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733465672781 2024-12-06T06:14:35,413 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 16fa05b2afd74719918e28234bd688b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733465673794 2024-12-06T06:14:35,413 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb92177835a04348a542bd6f7bcbc35e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733465673785 2024-12-06T06:14:35,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-06T06:14:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:35,416 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-06T06:14:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:35,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:35,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-06T06:14:35,432 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:35,433 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/99e75527ba8d41f9929dfe64678f40a3 is 50, key is test_row_0/B:col10/1733465673794/Put/seqid=0 2024-12-06T06:14:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b7aebcef8ed84213919ffdc30c4556b6_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465674451/Put/seqid=0 2024-12-06T06:14:35,442 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:35,461 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206b2b468edfa104f69bc0b12b3d7722eb8_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:35,462 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206b2b468edfa104f69bc0b12b3d7722eb8_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:35,463 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b2b468edfa104f69bc0b12b3d7722eb8_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:35,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742084_1260 (size=13187) 2024-12-06T06:14:35,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742085_1261 (size=12454) 2024-12-06T06:14:35,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742086_1262 (size=4469) 2024-12-06T06:14:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:35,496 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b7aebcef8ed84213919ffdc30c4556b6_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b7aebcef8ed84213919ffdc30c4556b6_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:35,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/184e8794c0d944749b1992427a91001d, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:35,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/184e8794c0d944749b1992427a91001d is 175, key is test_row_0/A:col10/1733465674451/Put/seqid=0 2024-12-06T06:14:35,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742087_1263 (size=31255) 2024-12-06T06:14:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:35,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. as already flushing 2024-12-06T06:14:35,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465675652, max timestamp=1733465675676Num uniques 859; 2024-12-06T06:14:35,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:14:35,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465735709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465735711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-06T06:14:35,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465735824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465735833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:35,885 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/99e75527ba8d41f9929dfe64678f40a3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e75527ba8d41f9929dfe64678f40a3 2024-12-06T06:14:35,889 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#220 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:35,890 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/5ab461682ef6433298cfba73fb17d0d1 is 175, key is test_row_0/A:col10/1733465673794/Put/seqid=0 2024-12-06T06:14:35,895 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into 99e75527ba8d41f9929dfe64678f40a3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:35,895 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:35,895 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=13, startTime=1733465675406; duration=0sec 2024-12-06T06:14:35,895 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:35,895 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:35,895 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:35,896 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:35,896 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:35,896 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:35,896 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/8d2f65b264d64ff882e06e02b80477c3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1d9b7710b03e4d578fb677320132bba7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/378c8961afad4bf08c8f53234e5cc1bf] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.8 K 2024-12-06T06:14:35,897 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2f65b264d64ff882e06e02b80477c3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733465671626 2024-12-06T06:14:35,897 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d9b7710b03e4d578fb677320132bba7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733465672781 2024-12-06T06:14:35,897 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 378c8961afad4bf08c8f53234e5cc1bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733465673794 2024-12-06T06:14:35,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742088_1264 (size=32141) 2024-12-06T06:14:35,904 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=479, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/184e8794c0d944749b1992427a91001d 2024-12-06T06:14:35,923 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:35,923 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/98122e72b2b8493f90e55b3bd3c8b6e4 is 50, key is test_row_0/C:col10/1733465673794/Put/seqid=0 2024-12-06T06:14:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/0e32e606536542fc9dd9f1596e4b09eb is 50, key is test_row_0/B:col10/1733465674451/Put/seqid=0 2024-12-06T06:14:35,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742089_1265 (size=13187) 2024-12-06T06:14:35,985 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/98122e72b2b8493f90e55b3bd3c8b6e4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/98122e72b2b8493f90e55b3bd3c8b6e4 2024-12-06T06:14:35,993 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into 98122e72b2b8493f90e55b3bd3c8b6e4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:35,993 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:35,993 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=13, startTime=1733465675406; duration=0sec 2024-12-06T06:14:35,993 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:35,993 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:36,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742090_1266 (size=12301) 2024-12-06T06:14:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465736034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:36,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465736044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-06T06:14:36,313 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/5ab461682ef6433298cfba73fb17d0d1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/5ab461682ef6433298cfba73fb17d0d1 2024-12-06T06:14:36,324 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into 5ab461682ef6433298cfba73fb17d0d1(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:36,324 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:36,324 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=13, startTime=1733465675406; duration=0sec 2024-12-06T06:14:36,324 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:36,324 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:36,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465736344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:36,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465736363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:36,404 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/0e32e606536542fc9dd9f1596e4b09eb 2024-12-06T06:14:36,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/cff47ab3dca1425eae003343ff2725fa is 50, key is test_row_0/C:col10/1733465674451/Put/seqid=0 2024-12-06T06:14:36,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742091_1267 (size=12301) 2024-12-06T06:14:36,454 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/cff47ab3dca1425eae003343ff2725fa 2024-12-06T06:14:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/184e8794c0d944749b1992427a91001d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/184e8794c0d944749b1992427a91001d 2024-12-06T06:14:36,465 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/184e8794c0d944749b1992427a91001d, entries=150, sequenceid=479, filesize=30.5 K 2024-12-06T06:14:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/0e32e606536542fc9dd9f1596e4b09eb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/0e32e606536542fc9dd9f1596e4b09eb 2024-12-06T06:14:36,473 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/0e32e606536542fc9dd9f1596e4b09eb, entries=150, sequenceid=479, filesize=12.0 K 2024-12-06T06:14:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/cff47ab3dca1425eae003343ff2725fa as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/cff47ab3dca1425eae003343ff2725fa 2024-12-06T06:14:36,481 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/cff47ab3dca1425eae003343ff2725fa, entries=150, sequenceid=479, filesize=12.0 K 2024-12-06T06:14:36,482 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=149.83 KB/153430 for efacad63d89edd72fefc481a1ce9f017 in 1066ms, sequenceid=479, compaction requested=false 2024-12-06T06:14:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-06T06:14:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-06T06:14:36,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-06T06:14:36,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3750 sec 2024-12-06T06:14:36,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.3830 sec 2024-12-06T06:14:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:36,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=156.54 KB heapSize=461.33 KB 2024-12-06T06:14:36,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:36,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:36,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:36,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:36,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:36,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:36,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53518 deadline: 1733465736873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:36,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465736875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:36,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f129f8982bc44469956e98aad644e8ec_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_0/A:col10/1733465675706/Put/seqid=0 2024-12-06T06:14:36,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742092_1268 (size=14994) 2024-12-06T06:14:36,901 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:36,909 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f129f8982bc44469956e98aad644e8ec_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f129f8982bc44469956e98aad644e8ec_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:36,912 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ec53564b46a745c1b4345cf05e5d2e3c, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:36,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ec53564b46a745c1b4345cf05e5d2e3c is 175, key is test_row_0/A:col10/1733465675706/Put/seqid=0 2024-12-06T06:14:36,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53580 deadline: 1733465736924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:36,928 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in efacad63d89edd72fefc481a1ce9f017 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18319 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:36,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742093_1269 (size=39949) 2024-12-06T06:14:36,942 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=513, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ec53564b46a745c1b4345cf05e5d2e3c 2024-12-06T06:14:36,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d40125bfe3494fb6bac53aa6bee77563 is 50, key is test_row_0/B:col10/1733465675706/Put/seqid=0 2024-12-06T06:14:36,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742094_1270 (size=12301) 2024-12-06T06:14:36,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d40125bfe3494fb6bac53aa6bee77563 2024-12-06T06:14:36,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/e8b39cc060fa4bf0bf22bbdf74dcfe33 is 50, key is test_row_0/C:col10/1733465675706/Put/seqid=0 2024-12-06T06:14:36,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1733465736985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:14:37,021 INFO [Thread-763 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-763. 2024-12-06T06:14:37,021 DEBUG [Thread-763 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:37,021 DEBUG [Thread-763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,021 DEBUG [Thread-763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:37,023 INFO [Thread-767 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-767. 2024-12-06T06:14:37,023 DEBUG [Thread-767 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:37,023 DEBUG [Thread-767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,023 DEBUG [Thread-767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:37,023 INFO [Thread-765 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-765. 2024-12-06T06:14:37,023 DEBUG [Thread-765 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:37,023 DEBUG [Thread-765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,023 DEBUG [Thread-765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:37,025 INFO [Thread-761 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-761. 2024-12-06T06:14:37,025 DEBUG [Thread-761 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:37,025 DEBUG [Thread-761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,025 DEBUG [Thread-761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:37,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742095_1271 (size=12301) 2024-12-06T06:14:37,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/e8b39cc060fa4bf0bf22bbdf74dcfe33 2024-12-06T06:14:37,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/ec53564b46a745c1b4345cf05e5d2e3c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ec53564b46a745c1b4345cf05e5d2e3c 2024-12-06T06:14:37,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ec53564b46a745c1b4345cf05e5d2e3c, entries=200, sequenceid=513, filesize=39.0 K 2024-12-06T06:14:37,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d40125bfe3494fb6bac53aa6bee77563 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d40125bfe3494fb6bac53aa6bee77563 2024-12-06T06:14:37,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d40125bfe3494fb6bac53aa6bee77563, entries=150, sequenceid=513, filesize=12.0 K 2024-12-06T06:14:37,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/e8b39cc060fa4bf0bf22bbdf74dcfe33 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e8b39cc060fa4bf0bf22bbdf74dcfe33 2024-12-06T06:14:37,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e8b39cc060fa4bf0bf22bbdf74dcfe33, entries=150, sequenceid=513, filesize=12.0 K 2024-12-06T06:14:37,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~156.54 KB/160300, heapSize ~461.28 KB/472352, currentSize=20.13 KB/20610 for efacad63d89edd72fefc481a1ce9f017 in 188ms, sequenceid=513, compaction requested=true 2024-12-06T06:14:37,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:37,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:37,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:37,047 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:37,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:37,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:37,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efacad63d89edd72fefc481a1ce9f017:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:37,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:37,047 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:37,048 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:37,048 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/A is initiating minor compaction (all files) 2024-12-06T06:14:37,048 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/A in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:37,048 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/5ab461682ef6433298cfba73fb17d0d1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/184e8794c0d944749b1992427a91001d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ec53564b46a745c1b4345cf05e5d2e3c] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=100.9 K 2024-12-06T06:14:37,048 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:37,049 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/5ab461682ef6433298cfba73fb17d0d1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/184e8794c0d944749b1992427a91001d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ec53564b46a745c1b4345cf05e5d2e3c] 2024-12-06T06:14:37,049 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:37,049 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/B is initiating minor compaction (all files) 2024-12-06T06:14:37,049 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ab461682ef6433298cfba73fb17d0d1, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733465673794 2024-12-06T06:14:37,049 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/B in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:37,049 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e75527ba8d41f9929dfe64678f40a3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/0e32e606536542fc9dd9f1596e4b09eb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d40125bfe3494fb6bac53aa6bee77563] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.9 K 2024-12-06T06:14:37,049 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 184e8794c0d944749b1992427a91001d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733465674438 2024-12-06T06:14:37,049 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e75527ba8d41f9929dfe64678f40a3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733465673794 2024-12-06T06:14:37,050 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec53564b46a745c1b4345cf05e5d2e3c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733465675698 2024-12-06T06:14:37,050 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e32e606536542fc9dd9f1596e4b09eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733465674438 2024-12-06T06:14:37,050 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d40125bfe3494fb6bac53aa6bee77563, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733465675706 2024-12-06T06:14:37,060 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#B#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:37,061 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/058497d684344b9a9bce34779edb1610 is 50, key is test_row_0/B:col10/1733465675706/Put/seqid=0 2024-12-06T06:14:37,065 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:37,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742096_1272 (size=13289) 2024-12-06T06:14:37,073 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412066a4c713b40e04f30af0c8325506135a4_efacad63d89edd72fefc481a1ce9f017 store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:37,103 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412066a4c713b40e04f30af0c8325506135a4_efacad63d89edd72fefc481a1ce9f017, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:37,103 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066a4c713b40e04f30af0c8325506135a4_efacad63d89edd72fefc481a1ce9f017 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:37,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742097_1273 (size=4469) 2024-12-06T06:14:37,198 INFO [Thread-750 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-750. 2024-12-06T06:14:37,198 DEBUG [Thread-750 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:37,198 DEBUG [Thread-750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,199 DEBUG [Thread-750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-06T06:14:37,254 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:37,476 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/058497d684344b9a9bce34779edb1610 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/058497d684344b9a9bce34779edb1610 2024-12-06T06:14:37,480 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/B of efacad63d89edd72fefc481a1ce9f017 into 058497d684344b9a9bce34779edb1610(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:37,480 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:37,480 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/B, priority=13, startTime=1733465677047; duration=0sec 2024-12-06T06:14:37,480 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:37,480 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:B 2024-12-06T06:14:37,481 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:37,481 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:37,481 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): efacad63d89edd72fefc481a1ce9f017/C is initiating minor compaction (all files) 2024-12-06T06:14:37,481 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efacad63d89edd72fefc481a1ce9f017/C in TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:37,482 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/98122e72b2b8493f90e55b3bd3c8b6e4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/cff47ab3dca1425eae003343ff2725fa, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e8b39cc060fa4bf0bf22bbdf74dcfe33] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp, totalSize=36.9 K 2024-12-06T06:14:37,482 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 98122e72b2b8493f90e55b3bd3c8b6e4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733465673794 2024-12-06T06:14:37,482 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting cff47ab3dca1425eae003343ff2725fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733465674438 2024-12-06T06:14:37,482 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b39cc060fa4bf0bf22bbdf74dcfe33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733465675706 2024-12-06T06:14:37,490 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#C#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:37,490 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/6decfe3f820a488d99dad7e170a0a469 is 50, key is test_row_0/C:col10/1733465675706/Put/seqid=0 2024-12-06T06:14:37,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742098_1274 (size=13289) 2024-12-06T06:14:37,515 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efacad63d89edd72fefc481a1ce9f017#A#compaction#228 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:37,516 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/261f0f9339ad4d34b0bfa66437cd96ce is 175, key is test_row_0/A:col10/1733465675706/Put/seqid=0 2024-12-06T06:14:37,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742099_1275 (size=32243) 2024-12-06T06:14:37,533 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/261f0f9339ad4d34b0bfa66437cd96ce as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/261f0f9339ad4d34b0bfa66437cd96ce 2024-12-06T06:14:37,539 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/A of efacad63d89edd72fefc481a1ce9f017 into 261f0f9339ad4d34b0bfa66437cd96ce(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:37,539 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:37,539 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/A, priority=13, startTime=1733465677047; duration=0sec 2024-12-06T06:14:37,539 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:37,539 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:A 2024-12-06T06:14:37,886 INFO [Thread-752 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-752. 2024-12-06T06:14:37,886 DEBUG [Thread-752 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:37,886 DEBUG [Thread-752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,886 DEBUG [Thread-752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:37,886 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:37,903 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/6decfe3f820a488d99dad7e170a0a469 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/6decfe3f820a488d99dad7e170a0a469 2024-12-06T06:14:37,908 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efacad63d89edd72fefc481a1ce9f017/C of efacad63d89edd72fefc481a1ce9f017 into 6decfe3f820a488d99dad7e170a0a469(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:37,908 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:37,908 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., storeName=efacad63d89edd72fefc481a1ce9f017/C, priority=13, startTime=1733465677047; duration=0sec 2024-12-06T06:14:37,908 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:37,908 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efacad63d89edd72fefc481a1ce9f017:C 2024-12-06T06:14:38,105 INFO [Thread-758 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-758. 2024-12-06T06:14:38,106 DEBUG [Thread-758 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:38,106 DEBUG [Thread-758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:38,106 DEBUG [Thread-758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:38,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:38,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:14:38,115 INFO [Thread-754 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-754. 2024-12-06T06:14:38,115 DEBUG [Thread-754 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:38,115 DEBUG [Thread-754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:38,115 DEBUG [Thread-754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:38,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:38,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:38,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:38,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064aa43da4e4a64a3d8a41a3aea33ddf68_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_1/A:col10/1733465678114/Put/seqid=0 2024-12-06T06:14:38,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742100_1276 (size=9914) 2024-12-06T06:14:38,525 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:38,529 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064aa43da4e4a64a3d8a41a3aea33ddf68_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064aa43da4e4a64a3d8a41a3aea33ddf68_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:38,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/8871a0f672a84945b40c7b9f214f94ac, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:38,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/8871a0f672a84945b40c7b9f214f94ac is 175, key is test_row_1/A:col10/1733465678114/Put/seqid=0 2024-12-06T06:14:38,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742101_1277 (size=22561) 2024-12-06T06:14:38,936 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=526, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/8871a0f672a84945b40c7b9f214f94ac 2024-12-06T06:14:38,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d431ed82be794fea865bfd31f9216168 is 50, key is test_row_1/B:col10/1733465678114/Put/seqid=0 2024-12-06T06:14:38,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742102_1278 (size=9857) 2024-12-06T06:14:39,290 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T06:14:39,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d431ed82be794fea865bfd31f9216168 2024-12-06T06:14:39,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/6294fe4e33344c7b8e9d85e52edf29d4 is 50, key is test_row_1/C:col10/1733465678114/Put/seqid=0 2024-12-06T06:14:39,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742103_1279 (size=9857) 2024-12-06T06:14:39,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/6294fe4e33344c7b8e9d85e52edf29d4 2024-12-06T06:14:39,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/8871a0f672a84945b40c7b9f214f94ac as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/8871a0f672a84945b40c7b9f214f94ac 2024-12-06T06:14:39,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/8871a0f672a84945b40c7b9f214f94ac, entries=100, sequenceid=526, filesize=22.0 K 2024-12-06T06:14:39,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/d431ed82be794fea865bfd31f9216168 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d431ed82be794fea865bfd31f9216168 2024-12-06T06:14:39,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d431ed82be794fea865bfd31f9216168, entries=100, sequenceid=526, filesize=9.6 K 2024-12-06T06:14:39,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/6294fe4e33344c7b8e9d85e52edf29d4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/6294fe4e33344c7b8e9d85e52edf29d4 2024-12-06T06:14:39,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/6294fe4e33344c7b8e9d85e52edf29d4, entries=100, sequenceid=526, filesize=9.6 K 2024-12-06T06:14:39,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for efacad63d89edd72fefc481a1ce9f017 in 1267ms, sequenceid=526, compaction requested=false 2024-12-06T06:14:39,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efacad63d89edd72fefc481a1ce9f017: 2024-12-06T06:14:46,966 INFO [Thread-756 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-756. 2024-12-06T06:14:46,966 DEBUG [Thread-756 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:14:46,966 DEBUG [Thread-756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:46,966 DEBUG [Thread-756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:46,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2364 2024-12-06T06:14:46,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2501 2024-12-06T06:14:46,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T06:14:46,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1077 2024-12-06T06:14:46,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3231 rows 2024-12-06T06:14:46,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1088 2024-12-06T06:14:46,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3261 rows 2024-12-06T06:14:46,967 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T06:14:46,967 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:14:46,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:46,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:46,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:14:46,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T06:14:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T06:14:46,971 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465686971"}]},"ts":"1733465686971"} 2024-12-06T06:14:46,972 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T06:14:46,972 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T06:14:46,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:14:46,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, UNASSIGN}] 2024-12-06T06:14:46,975 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, UNASSIGN 2024-12-06T06:14:46,976 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=efacad63d89edd72fefc481a1ce9f017, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:46,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, UNASSIGN because future has completed 2024-12-06T06:14:46,977 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:14:46,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T06:14:47,129 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:47,130 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:14:47,130 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing efacad63d89edd72fefc481a1ce9f017, disabling compactions & flushes 2024-12-06T06:14:47,130 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:47,130 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:47,130 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. after waiting 0 ms 2024-12-06T06:14:47,130 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:47,130 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing efacad63d89edd72fefc481a1ce9f017 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-06T06:14:47,130 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=A 2024-12-06T06:14:47,130 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:47,131 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=B 2024-12-06T06:14:47,131 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:47,131 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efacad63d89edd72fefc481a1ce9f017, store=C 2024-12-06T06:14:47,131 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:47,137 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206bdfc74611e834109beab219b5b25e655_efacad63d89edd72fefc481a1ce9f017 is 50, key is test_row_2/A:col10/1733465686964/Put/seqid=0 2024-12-06T06:14:47,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742104_1280 (size=7374) 2024-12-06T06:14:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T06:14:47,541 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:47,545 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206bdfc74611e834109beab219b5b25e655_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bdfc74611e834109beab219b5b25e655_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:47,546 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/eab2c192120946b3ade5ac7c2dc56490, store: [table=TestAcidGuarantees family=A region=efacad63d89edd72fefc481a1ce9f017] 2024-12-06T06:14:47,547 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/eab2c192120946b3ade5ac7c2dc56490 is 175, key is test_row_2/A:col10/1733465686964/Put/seqid=0 2024-12-06T06:14:47,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742105_1281 (size=13865) 2024-12-06T06:14:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T06:14:47,951 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=530, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/eab2c192120946b3ade5ac7c2dc56490 2024-12-06T06:14:47,957 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/5fb984755ebb496f9c34985496820cec is 50, key is test_row_2/B:col10/1733465686964/Put/seqid=0 2024-12-06T06:14:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742106_1282 (size=7415) 2024-12-06T06:14:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T06:14:48,361 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/5fb984755ebb496f9c34985496820cec 2024-12-06T06:14:48,367 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b27b4af12f1b446bbb9ef87fe8de2fc5 is 50, key is test_row_2/C:col10/1733465686964/Put/seqid=0 2024-12-06T06:14:48,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742107_1283 (size=7415) 2024-12-06T06:14:48,771 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b27b4af12f1b446bbb9ef87fe8de2fc5 2024-12-06T06:14:48,775 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/A/eab2c192120946b3ade5ac7c2dc56490 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/eab2c192120946b3ade5ac7c2dc56490 2024-12-06T06:14:48,778 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/eab2c192120946b3ade5ac7c2dc56490, entries=50, sequenceid=530, filesize=13.5 K 2024-12-06T06:14:48,779 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/B/5fb984755ebb496f9c34985496820cec as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5fb984755ebb496f9c34985496820cec 2024-12-06T06:14:48,782 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5fb984755ebb496f9c34985496820cec, entries=50, sequenceid=530, filesize=7.2 K 2024-12-06T06:14:48,783 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/.tmp/C/b27b4af12f1b446bbb9ef87fe8de2fc5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b27b4af12f1b446bbb9ef87fe8de2fc5 2024-12-06T06:14:48,786 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b27b4af12f1b446bbb9ef87fe8de2fc5, entries=50, sequenceid=530, filesize=7.2 K 2024-12-06T06:14:48,787 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for efacad63d89edd72fefc481a1ce9f017 in 1657ms, sequenceid=530, compaction requested=true 2024-12-06T06:14:48,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/a2d1bd685ec540358cd59ddccf32d280, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa7de25415ce4129b15726d6b9d179f8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9ff6993d1b30492db2c2fc84acbea254, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ca0e3be655dc4d77b36ca17d970f9379, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/6eded4bb8be542e09150fea962adbab2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa91538e2e0a4f7b91d8c9beda79cf22, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/98c2f98efeff4ac78c4984e52742304b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/42f07c5bac314ec4bb80764734bdcb3e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fdf162b9edb4481988efda588f358057, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b0a8b4ecd3da4ea3a4613ca9bf7a2941, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7647e908e3b347c084fd51048c63391a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/1c3f1d931e3d46438d520db13ecd2707, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d468e2b0525140a395d57f08d68c3fc5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/c64c2d01152b4e0398c576069004ddd9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d43d4fb725f54e33a0fd69dede735629, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/18736f4b4f6e4560bda796c1ee461a38, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b7c525b9caf14977829997c696d21cd3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/bec32f0d6375484fa92950fc60593f4e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cc4174d44cce4271946a7d1fb130cfde, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ad7fdac278fe4bb883c3efc01679cfe6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7ad019894ef949c8a8db549223fbe374, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/174b07a60c4943929c769d93b8a45075, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/3e19d7be698f467ea5098259e0bc2cd1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9bd87c0e39fb4b4da6945752fbf3eba7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/e71c8b0082864a3eb6da883311fa029c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cb92177835a04348a542bd6f7bcbc35e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/5ab461682ef6433298cfba73fb17d0d1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/184e8794c0d944749b1992427a91001d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ec53564b46a745c1b4345cf05e5d2e3c] to archive 2024-12-06T06:14:48,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:14:48,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/a2d1bd685ec540358cd59ddccf32d280 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/a2d1bd685ec540358cd59ddccf32d280 2024-12-06T06:14:48,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa7de25415ce4129b15726d6b9d179f8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa7de25415ce4129b15726d6b9d179f8 2024-12-06T06:14:48,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9ff6993d1b30492db2c2fc84acbea254 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9ff6993d1b30492db2c2fc84acbea254 2024-12-06T06:14:48,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ca0e3be655dc4d77b36ca17d970f9379 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ca0e3be655dc4d77b36ca17d970f9379 2024-12-06T06:14:48,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/6eded4bb8be542e09150fea962adbab2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/6eded4bb8be542e09150fea962adbab2 2024-12-06T06:14:48,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa91538e2e0a4f7b91d8c9beda79cf22 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fa91538e2e0a4f7b91d8c9beda79cf22 2024-12-06T06:14:48,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/98c2f98efeff4ac78c4984e52742304b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/98c2f98efeff4ac78c4984e52742304b 2024-12-06T06:14:48,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/42f07c5bac314ec4bb80764734bdcb3e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/42f07c5bac314ec4bb80764734bdcb3e 2024-12-06T06:14:48,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fdf162b9edb4481988efda588f358057 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/fdf162b9edb4481988efda588f358057 2024-12-06T06:14:48,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b0a8b4ecd3da4ea3a4613ca9bf7a2941 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b0a8b4ecd3da4ea3a4613ca9bf7a2941 2024-12-06T06:14:48,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7647e908e3b347c084fd51048c63391a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7647e908e3b347c084fd51048c63391a 2024-12-06T06:14:48,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/1c3f1d931e3d46438d520db13ecd2707 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/1c3f1d931e3d46438d520db13ecd2707 2024-12-06T06:14:48,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d468e2b0525140a395d57f08d68c3fc5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d468e2b0525140a395d57f08d68c3fc5 2024-12-06T06:14:48,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/c64c2d01152b4e0398c576069004ddd9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/c64c2d01152b4e0398c576069004ddd9 2024-12-06T06:14:48,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d43d4fb725f54e33a0fd69dede735629 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/d43d4fb725f54e33a0fd69dede735629 2024-12-06T06:14:48,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/18736f4b4f6e4560bda796c1ee461a38 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/18736f4b4f6e4560bda796c1ee461a38 2024-12-06T06:14:48,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b7c525b9caf14977829997c696d21cd3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/b7c525b9caf14977829997c696d21cd3 2024-12-06T06:14:48,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/bec32f0d6375484fa92950fc60593f4e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/bec32f0d6375484fa92950fc60593f4e 2024-12-06T06:14:48,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cc4174d44cce4271946a7d1fb130cfde to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cc4174d44cce4271946a7d1fb130cfde 2024-12-06T06:14:48,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ad7fdac278fe4bb883c3efc01679cfe6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ad7fdac278fe4bb883c3efc01679cfe6 2024-12-06T06:14:48,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7ad019894ef949c8a8db549223fbe374 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/7ad019894ef949c8a8db549223fbe374 2024-12-06T06:14:48,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/174b07a60c4943929c769d93b8a45075 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/174b07a60c4943929c769d93b8a45075 2024-12-06T06:14:48,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/3e19d7be698f467ea5098259e0bc2cd1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/3e19d7be698f467ea5098259e0bc2cd1 2024-12-06T06:14:48,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9bd87c0e39fb4b4da6945752fbf3eba7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/9bd87c0e39fb4b4da6945752fbf3eba7 2024-12-06T06:14:48,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/e71c8b0082864a3eb6da883311fa029c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/e71c8b0082864a3eb6da883311fa029c 2024-12-06T06:14:48,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cb92177835a04348a542bd6f7bcbc35e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/cb92177835a04348a542bd6f7bcbc35e 2024-12-06T06:14:48,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/5ab461682ef6433298cfba73fb17d0d1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/5ab461682ef6433298cfba73fb17d0d1 2024-12-06T06:14:48,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/184e8794c0d944749b1992427a91001d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/184e8794c0d944749b1992427a91001d 2024-12-06T06:14:48,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ec53564b46a745c1b4345cf05e5d2e3c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/ec53564b46a745c1b4345cf05e5d2e3c 2024-12-06T06:14:48,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/a5f9d45634744f6d8ef29284d227dbc4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d2781b4a56ef4ed193c325383d450d12, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/69ac9f6db4564f7fa25f3317628912d2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/bfc96b07e94f435b972e3e497197c574, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d845b290e3464d96a8347d631950ed90, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/ecbb8db9496f4d7aa573e09e37ac1187, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/c5e8fa12dd2f4edc803b2476cd52d48a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/85745b1ecea74a149b70823babb9e72d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d651846ac9ec4c7daca796f5c1420c1e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/08a2f375fe9c41818502ad7eaa4b2d8b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16312bbdf6434b05bc0ac639863df40f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cb26b446304a4b23a01833c07daa5669, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/07db8f3a276b452c96abacf91801f3ad, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cf64754db0f04e748b667208bff05ec9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/8c921d59db1546ee99682a3bae5511a0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e689e5ffb341128e898bd3de0f78d2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/9b94f5f7c2154dbabda13a813b40be67, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/117299a7db2e4173a4f66881c37888df, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/3f4946ae752a426eab256beb39948ea6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5f6767a8490c4ac38268766fe4e48fb0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/b0ce051dd3924dd0aa4049ab6599b2e6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/e0310be5312c4e4ab8cff8953d2f0540, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/973689f9467f4344a07a18dbab66580f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/956e4abe84404dd7908fd11ac13c5f29, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/f260bfdf48e24aa981764bf1ff19a32c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e75527ba8d41f9929dfe64678f40a3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16fa05b2afd74719918e28234bd688b7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/0e32e606536542fc9dd9f1596e4b09eb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d40125bfe3494fb6bac53aa6bee77563] to archive 2024-12-06T06:14:48,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:14:48,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/a5f9d45634744f6d8ef29284d227dbc4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/a5f9d45634744f6d8ef29284d227dbc4 2024-12-06T06:14:48,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d2781b4a56ef4ed193c325383d450d12 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d2781b4a56ef4ed193c325383d450d12 2024-12-06T06:14:48,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/69ac9f6db4564f7fa25f3317628912d2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/69ac9f6db4564f7fa25f3317628912d2 2024-12-06T06:14:48,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/bfc96b07e94f435b972e3e497197c574 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/bfc96b07e94f435b972e3e497197c574 2024-12-06T06:14:48,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d845b290e3464d96a8347d631950ed90 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d845b290e3464d96a8347d631950ed90 2024-12-06T06:14:48,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/ecbb8db9496f4d7aa573e09e37ac1187 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/ecbb8db9496f4d7aa573e09e37ac1187 2024-12-06T06:14:48,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/c5e8fa12dd2f4edc803b2476cd52d48a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/c5e8fa12dd2f4edc803b2476cd52d48a 2024-12-06T06:14:48,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/85745b1ecea74a149b70823babb9e72d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/85745b1ecea74a149b70823babb9e72d 2024-12-06T06:14:48,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d651846ac9ec4c7daca796f5c1420c1e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d651846ac9ec4c7daca796f5c1420c1e 2024-12-06T06:14:48,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/08a2f375fe9c41818502ad7eaa4b2d8b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/08a2f375fe9c41818502ad7eaa4b2d8b 2024-12-06T06:14:48,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16312bbdf6434b05bc0ac639863df40f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16312bbdf6434b05bc0ac639863df40f 2024-12-06T06:14:48,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cb26b446304a4b23a01833c07daa5669 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cb26b446304a4b23a01833c07daa5669 2024-12-06T06:14:48,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/07db8f3a276b452c96abacf91801f3ad to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/07db8f3a276b452c96abacf91801f3ad 2024-12-06T06:14:48,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cf64754db0f04e748b667208bff05ec9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/cf64754db0f04e748b667208bff05ec9 2024-12-06T06:14:48,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/8c921d59db1546ee99682a3bae5511a0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/8c921d59db1546ee99682a3bae5511a0 2024-12-06T06:14:48,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e689e5ffb341128e898bd3de0f78d2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e689e5ffb341128e898bd3de0f78d2 2024-12-06T06:14:48,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/9b94f5f7c2154dbabda13a813b40be67 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/9b94f5f7c2154dbabda13a813b40be67 2024-12-06T06:14:48,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/117299a7db2e4173a4f66881c37888df to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/117299a7db2e4173a4f66881c37888df 2024-12-06T06:14:48,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/3f4946ae752a426eab256beb39948ea6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/3f4946ae752a426eab256beb39948ea6 2024-12-06T06:14:48,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5f6767a8490c4ac38268766fe4e48fb0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5f6767a8490c4ac38268766fe4e48fb0 2024-12-06T06:14:48,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/b0ce051dd3924dd0aa4049ab6599b2e6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/b0ce051dd3924dd0aa4049ab6599b2e6 2024-12-06T06:14:48,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/e0310be5312c4e4ab8cff8953d2f0540 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/e0310be5312c4e4ab8cff8953d2f0540 2024-12-06T06:14:48,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/973689f9467f4344a07a18dbab66580f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/973689f9467f4344a07a18dbab66580f 2024-12-06T06:14:48,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/956e4abe84404dd7908fd11ac13c5f29 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/956e4abe84404dd7908fd11ac13c5f29 2024-12-06T06:14:48,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/f260bfdf48e24aa981764bf1ff19a32c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/f260bfdf48e24aa981764bf1ff19a32c 2024-12-06T06:14:48,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e75527ba8d41f9929dfe64678f40a3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/99e75527ba8d41f9929dfe64678f40a3 2024-12-06T06:14:48,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16fa05b2afd74719918e28234bd688b7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/16fa05b2afd74719918e28234bd688b7 2024-12-06T06:14:48,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/0e32e606536542fc9dd9f1596e4b09eb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/0e32e606536542fc9dd9f1596e4b09eb 2024-12-06T06:14:48,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d40125bfe3494fb6bac53aa6bee77563 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d40125bfe3494fb6bac53aa6bee77563 2024-12-06T06:14:48,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d8636115608e4357a5a8bccac9ce779a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b9bcf146afda4cd199f68b28f67e466d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/06c1249c8faa49d0b90f9ad341c58834, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/39f352ca161246afaf557e2d3ef7ca19, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a9b54ab94d7c49f1a2d387537ffaf171, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e36ef84d17bf463a8d9666633f64e3a3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d9707f7fcb0040dfa3d3e673eefbd5ec, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a8453a7250a748b9848efc5a7a9d0e3f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/89df661ccf14479d85771b3e1bdfad7a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1a1e21b3e9524984ab5ab431af022279, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/5d7f800dd48d43dd8756eae52b50a885, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/c70d15958f744c3a8556f6c843fbee5d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b8ca3051056c45f5b0e251f532022330, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b2ea04f690bd4482a85aac23d8cc511c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/64254ecb00644e1a91ce6cad166aa1a9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/bdfa597d947a4202a1575ea6a1f89cec, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a2e6fa1c515944229e1fa116716e9614, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/3298ac5f8e064efa961a185ee7c58ef3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/162b35929a374bcca94f650525c71f5d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f63e4678866a42ceb12f8520d8a6c1b5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/66efe8bb7c954c9c8bc4d227f44c2420, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f03f07ec01f44ec2a62fa39eb6a9a605, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/8d2f65b264d64ff882e06e02b80477c3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/29ac70db79af49ee8e6dde9e3eb1b3e2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1d9b7710b03e4d578fb677320132bba7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/98122e72b2b8493f90e55b3bd3c8b6e4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/378c8961afad4bf08c8f53234e5cc1bf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/cff47ab3dca1425eae003343ff2725fa, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e8b39cc060fa4bf0bf22bbdf74dcfe33] to archive 2024-12-06T06:14:48,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:14:48,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d8636115608e4357a5a8bccac9ce779a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d8636115608e4357a5a8bccac9ce779a 2024-12-06T06:14:48,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b9bcf146afda4cd199f68b28f67e466d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b9bcf146afda4cd199f68b28f67e466d 2024-12-06T06:14:48,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/06c1249c8faa49d0b90f9ad341c58834 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/06c1249c8faa49d0b90f9ad341c58834 2024-12-06T06:14:48,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/39f352ca161246afaf557e2d3ef7ca19 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/39f352ca161246afaf557e2d3ef7ca19 2024-12-06T06:14:48,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a9b54ab94d7c49f1a2d387537ffaf171 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a9b54ab94d7c49f1a2d387537ffaf171 2024-12-06T06:14:48,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e36ef84d17bf463a8d9666633f64e3a3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e36ef84d17bf463a8d9666633f64e3a3 2024-12-06T06:14:48,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d9707f7fcb0040dfa3d3e673eefbd5ec to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/d9707f7fcb0040dfa3d3e673eefbd5ec 2024-12-06T06:14:48,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a8453a7250a748b9848efc5a7a9d0e3f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a8453a7250a748b9848efc5a7a9d0e3f 2024-12-06T06:14:48,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/89df661ccf14479d85771b3e1bdfad7a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/89df661ccf14479d85771b3e1bdfad7a 2024-12-06T06:14:48,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1a1e21b3e9524984ab5ab431af022279 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1a1e21b3e9524984ab5ab431af022279 2024-12-06T06:14:48,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/5d7f800dd48d43dd8756eae52b50a885 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/5d7f800dd48d43dd8756eae52b50a885 2024-12-06T06:14:48,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/c70d15958f744c3a8556f6c843fbee5d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/c70d15958f744c3a8556f6c843fbee5d 2024-12-06T06:14:48,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b8ca3051056c45f5b0e251f532022330 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b8ca3051056c45f5b0e251f532022330 2024-12-06T06:14:48,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b2ea04f690bd4482a85aac23d8cc511c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b2ea04f690bd4482a85aac23d8cc511c 2024-12-06T06:14:48,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/64254ecb00644e1a91ce6cad166aa1a9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/64254ecb00644e1a91ce6cad166aa1a9 2024-12-06T06:14:48,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/bdfa597d947a4202a1575ea6a1f89cec to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/bdfa597d947a4202a1575ea6a1f89cec 2024-12-06T06:14:48,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a2e6fa1c515944229e1fa116716e9614 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/a2e6fa1c515944229e1fa116716e9614 2024-12-06T06:14:48,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/3298ac5f8e064efa961a185ee7c58ef3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/3298ac5f8e064efa961a185ee7c58ef3 2024-12-06T06:14:48,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/162b35929a374bcca94f650525c71f5d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/162b35929a374bcca94f650525c71f5d 2024-12-06T06:14:48,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f63e4678866a42ceb12f8520d8a6c1b5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f63e4678866a42ceb12f8520d8a6c1b5 2024-12-06T06:14:48,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/66efe8bb7c954c9c8bc4d227f44c2420 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/66efe8bb7c954c9c8bc4d227f44c2420 2024-12-06T06:14:48,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f03f07ec01f44ec2a62fa39eb6a9a605 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/f03f07ec01f44ec2a62fa39eb6a9a605 2024-12-06T06:14:48,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/8d2f65b264d64ff882e06e02b80477c3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/8d2f65b264d64ff882e06e02b80477c3 2024-12-06T06:14:48,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/29ac70db79af49ee8e6dde9e3eb1b3e2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/29ac70db79af49ee8e6dde9e3eb1b3e2 2024-12-06T06:14:48,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1d9b7710b03e4d578fb677320132bba7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/1d9b7710b03e4d578fb677320132bba7 2024-12-06T06:14:48,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/98122e72b2b8493f90e55b3bd3c8b6e4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/98122e72b2b8493f90e55b3bd3c8b6e4 2024-12-06T06:14:48,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/378c8961afad4bf08c8f53234e5cc1bf to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/378c8961afad4bf08c8f53234e5cc1bf 2024-12-06T06:14:48,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/cff47ab3dca1425eae003343ff2725fa to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/cff47ab3dca1425eae003343ff2725fa 2024-12-06T06:14:48,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e8b39cc060fa4bf0bf22bbdf74dcfe33 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/e8b39cc060fa4bf0bf22bbdf74dcfe33 2024-12-06T06:14:48,897 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/recovered.edits/533.seqid, newMaxSeqId=533, maxSeqId=4 2024-12-06T06:14:48,898 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. 2024-12-06T06:14:48,898 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for efacad63d89edd72fefc481a1ce9f017: Waiting for close lock at 1733465687130Running coprocessor pre-close hooks at 1733465687130Disabling compacts and flushes for region at 1733465687130Disabling writes for close at 1733465687130Obtaining lock to block concurrent updates at 1733465687130Preparing flush snapshotting stores in efacad63d89edd72fefc481a1ce9f017 at 1733465687130Finished memstore snapshotting TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1733465687131 (+1 ms)Flushing stores of TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017. at 1733465687131Flushing efacad63d89edd72fefc481a1ce9f017/A: creating writer at 1733465687131Flushing mob file efacad63d89edd72fefc481a1ce9f017/A: appending metadata at 1733465687137 (+6 ms)Flushing mob file efacad63d89edd72fefc481a1ce9f017/A: closing flushed file at 1733465687137Flushing efacad63d89edd72fefc481a1ce9f017/A: appending metadata at 1733465687546 (+409 ms)Flushing efacad63d89edd72fefc481a1ce9f017/A: closing flushed file at 1733465687546Flushing efacad63d89edd72fefc481a1ce9f017/B: creating writer at 1733465687954 (+408 ms)Flushing efacad63d89edd72fefc481a1ce9f017/B: appending metadata at 1733465687957 (+3 ms)Flushing efacad63d89edd72fefc481a1ce9f017/B: closing flushed file at 1733465687957Flushing efacad63d89edd72fefc481a1ce9f017/C: creating writer at 1733465688364 (+407 ms)Flushing efacad63d89edd72fefc481a1ce9f017/C: appending metadata at 1733465688367 (+3 ms)Flushing efacad63d89edd72fefc481a1ce9f017/C: closing flushed file at 1733465688367Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e8862ac: reopening flushed file at 1733465688774 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f6c749: reopening flushed file at 1733465688779 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@277b58a9: reopening flushed file at 1733465688783 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for efacad63d89edd72fefc481a1ce9f017 in 1657ms, sequenceid=530, compaction requested=true at 1733465688787 (+4 ms)Writing region close event to WAL at 1733465688893 (+106 ms)Running coprocessor post-close hooks at 1733465688898 (+5 ms)Closed at 1733465688898 2024-12-06T06:14:48,900 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:48,901 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=efacad63d89edd72fefc481a1ce9f017, regionState=CLOSED 2024-12-06T06:14:48,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:14:48,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-06T06:14:48,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure efacad63d89edd72fefc481a1ce9f017, server=6336e8474e1c,39987,1733465620995 in 1.9260 sec 2024-12-06T06:14:48,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-06T06:14:48,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efacad63d89edd72fefc481a1ce9f017, UNASSIGN in 1.9310 sec 2024-12-06T06:14:48,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-06T06:14:48,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9340 sec 2024-12-06T06:14:48,910 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465688910"}]},"ts":"1733465688910"} 2024-12-06T06:14:48,911 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T06:14:48,912 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T06:14:48,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9440 sec 2024-12-06T06:14:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T06:14:49,114 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:49,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T06:14:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:49,117 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T06:14:49,118 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:49,121 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,123 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/recovered.edits] 2024-12-06T06:14:49,126 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/261f0f9339ad4d34b0bfa66437cd96ce to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/261f0f9339ad4d34b0bfa66437cd96ce 2024-12-06T06:14:49,127 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/8871a0f672a84945b40c7b9f214f94ac to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/8871a0f672a84945b40c7b9f214f94ac 2024-12-06T06:14:49,129 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/eab2c192120946b3ade5ac7c2dc56490 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/A/eab2c192120946b3ade5ac7c2dc56490 2024-12-06T06:14:49,134 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/058497d684344b9a9bce34779edb1610 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/058497d684344b9a9bce34779edb1610 2024-12-06T06:14:49,137 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5fb984755ebb496f9c34985496820cec to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/5fb984755ebb496f9c34985496820cec 2024-12-06T06:14:49,138 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d431ed82be794fea865bfd31f9216168 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/B/d431ed82be794fea865bfd31f9216168 2024-12-06T06:14:49,141 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/6294fe4e33344c7b8e9d85e52edf29d4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/6294fe4e33344c7b8e9d85e52edf29d4 2024-12-06T06:14:49,143 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/6decfe3f820a488d99dad7e170a0a469 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/6decfe3f820a488d99dad7e170a0a469 2024-12-06T06:14:49,144 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b27b4af12f1b446bbb9ef87fe8de2fc5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/C/b27b4af12f1b446bbb9ef87fe8de2fc5 2024-12-06T06:14:49,148 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/recovered.edits/533.seqid to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017/recovered.edits/533.seqid 2024-12-06T06:14:49,148 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,149 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T06:14:49,149 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T06:14:49,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T06:14:49,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060184b5f03cf24162bdc590c2b4441748_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060184b5f03cf24162bdc590c2b4441748_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,158 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120617a30a05674d462a939203bd6895a092_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120617a30a05674d462a939203bd6895a092_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061ec43441427246508a38d8fe022757a4_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061ec43441427246508a38d8fe022757a4_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,161 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206204ff93810de40008e58a434453a654a_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206204ff93810de40008e58a434453a654a_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206215c1f7a55ce4faabf0ab5d165574868_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206215c1f7a55ce4faabf0ab5d165574868_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,165 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120622b271d2fccf49e3b3585198a04886f0_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120622b271d2fccf49e3b3585198a04886f0_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,169 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206287f48cc1bfc4c8fba60ca908fa9be90_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206287f48cc1bfc4c8fba60ca908fa9be90_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,171 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b0ac4209575429180ccfd3bbbaa398c_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b0ac4209575429180ccfd3bbbaa398c_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,172 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120631e92748efcf4767b15ab97b7c478a92_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120631e92748efcf4767b15ab97b7c478a92_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,175 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063fb23419fb3846aabe5956ec5856367e_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063fb23419fb3846aabe5956ec5856367e_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,176 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064aa43da4e4a64a3d8a41a3aea33ddf68_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064aa43da4e4a64a3d8a41a3aea33ddf68_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,178 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120655330a8bd9d24ba595a1c4f11aeec735_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120655330a8bd9d24ba595a1c4f11aeec735_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,179 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067cab84fd41454522ab2436b8e80e6c39_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067cab84fd41454522ab2436b8e80e6c39_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,187 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120688185c3820aa49f1a75ceac97829f55f_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120688185c3820aa49f1a75ceac97829f55f_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,189 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206947a4b2122814c558faddd54557ba3a5_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206947a4b2122814c558faddd54557ba3a5_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,190 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b157f8f578824a8d96c2349c3af8a2bd_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b157f8f578824a8d96c2349c3af8a2bd_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,192 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b1ef4ed4151d4013961e629021d2a807_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b1ef4ed4151d4013961e629021d2a807_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,194 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b7aebcef8ed84213919ffdc30c4556b6_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b7aebcef8ed84213919ffdc30c4556b6_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,195 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bdfc74611e834109beab219b5b25e655_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bdfc74611e834109beab219b5b25e655_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,197 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c7d085c5608a49aaabdf41df1d3e3dc0_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c7d085c5608a49aaabdf41df1d3e3dc0_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,198 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e054128a810d4727849155283ebf7c0a_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206e054128a810d4727849155283ebf7c0a_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,200 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eace28d72dc64caf9636747ec30fb533_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eace28d72dc64caf9636747ec30fb533_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,201 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f129f8982bc44469956e98aad644e8ec_efacad63d89edd72fefc481a1ce9f017 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f129f8982bc44469956e98aad644e8ec_efacad63d89edd72fefc481a1ce9f017 2024-12-06T06:14:49,202 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T06:14:49,204 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:49,211 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T06:14:49,214 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T06:14:49,215 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:49,215 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T06:14:49,216 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733465689215"}]},"ts":"9223372036854775807"} 2024-12-06T06:14:49,218 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T06:14:49,218 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => efacad63d89edd72fefc481a1ce9f017, NAME => 'TestAcidGuarantees,,1733465653858.efacad63d89edd72fefc481a1ce9f017.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T06:14:49,219 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T06:14:49,219 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733465689219"}]},"ts":"9223372036854775807"} 2024-12-06T06:14:49,221 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T06:14:49,222 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:49,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 107 msec 2024-12-06T06:14:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T06:14:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T06:14:49,224 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:49,240 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=435 (was 388) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5084 (was 5514) 2024-12-06T06:14:49,254 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=435, ProcessCount=11, AvailableMemoryMB=5077 2024-12-06T06:14:49,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:14:49,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:14:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T06:14:49,258 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T06:14:49,258 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:49,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-06T06:14:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-06T06:14:49,259 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T06:14:49,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742108_1284 (size=1061) 2024-12-06T06:14:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-06T06:14:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-06T06:14:49,669 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea 2024-12-06T06:14:49,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742109_1285 (size=53) 2024-12-06T06:14:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-06T06:14:50,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:14:50,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aa1db8fd546545da53b24cabfd6bfcf1, disabling compactions & flushes 2024-12-06T06:14:50,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:50,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:50,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. after waiting 0 ms 2024-12-06T06:14:50,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:50,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:50,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa1db8fd546545da53b24cabfd6bfcf1: Waiting for close lock at 1733465690076Disabling compacts and flushes for region at 1733465690076Disabling writes for close at 1733465690076Writing region close event to WAL at 1733465690076Closed at 1733465690076 2024-12-06T06:14:50,078 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T06:14:50,078 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733465690078"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733465690078"}]},"ts":"1733465690078"} 2024-12-06T06:14:50,079 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T06:14:50,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T06:14:50,080 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465690080"}]},"ts":"1733465690080"} 2024-12-06T06:14:50,081 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T06:14:50,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, ASSIGN}] 2024-12-06T06:14:50,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, ASSIGN 2024-12-06T06:14:50,083 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, ASSIGN; state=OFFLINE, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=false 2024-12-06T06:14:50,234 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=aa1db8fd546545da53b24cabfd6bfcf1, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:50,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, ASSIGN because future has completed 2024-12-06T06:14:50,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:14:50,391 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:50,391 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:14:50,392 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,392 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:14:50,392 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,392 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,393 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-06T06:14:50,394 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:50,395 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa1db8fd546545da53b24cabfd6bfcf1 columnFamilyName A 2024-12-06T06:14:50,395 DEBUG [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:50,395 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.HStore(327): Store=aa1db8fd546545da53b24cabfd6bfcf1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:50,395 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,396 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:50,397 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa1db8fd546545da53b24cabfd6bfcf1 columnFamilyName B 2024-12-06T06:14:50,397 DEBUG [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:50,397 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.HStore(327): Store=aa1db8fd546545da53b24cabfd6bfcf1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:50,397 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,398 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:14:50,398 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa1db8fd546545da53b24cabfd6bfcf1 columnFamilyName C 2024-12-06T06:14:50,398 DEBUG [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:14:50,399 INFO [StoreOpener-aa1db8fd546545da53b24cabfd6bfcf1-1 {}] regionserver.HStore(327): Store=aa1db8fd546545da53b24cabfd6bfcf1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:14:50,399 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:50,399 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,400 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,400 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,401 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,401 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,401 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:14:50,402 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,405 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T06:14:50,405 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened aa1db8fd546545da53b24cabfd6bfcf1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60279057, jitterRate=-0.10177205502986908}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:14:50,405 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:50,406 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for aa1db8fd546545da53b24cabfd6bfcf1: Running coprocessor pre-open hook at 1733465690392Writing region info on filesystem at 1733465690392Initializing all the Stores at 1733465690393 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465690393Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465690393Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465690393Cleaning up temporary data from old regions at 1733465690401 (+8 ms)Running coprocessor post-open hooks at 1733465690405 (+4 ms)Region opened successfully at 1733465690406 (+1 ms) 2024-12-06T06:14:50,407 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., pid=65, masterSystemTime=1733465690388 2024-12-06T06:14:50,409 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:50,409 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:50,409 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=aa1db8fd546545da53b24cabfd6bfcf1, regionState=OPEN, openSeqNum=2, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:50,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:14:50,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-06T06:14:50,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 in 176 msec 2024-12-06T06:14:50,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-06T06:14:50,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, ASSIGN in 333 msec 2024-12-06T06:14:50,416 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T06:14:50,416 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465690416"}]},"ts":"1733465690416"} 2024-12-06T06:14:50,418 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T06:14:50,419 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T06:14:50,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-12-06T06:14:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-06T06:14:51,404 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:51,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dfd6f0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fbbbde8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,408 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,411 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T06:14:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 6336e8474e1c,40959,1733465620259 2024-12-06T06:14:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a041457 2024-12-06T06:14:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T06:14:51,413 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T06:14:51,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,414 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b70e9fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,416 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7799111a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd8e461, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52410290, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,420 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71efea32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,421 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636db275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,423 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc5a836, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,424 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ed32d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,426 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eaf402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,427 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d85a4a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,428 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61132800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,429 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364d3028, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,431 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743d8a08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea25f20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd77c81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a401cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc00bc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,437 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b163f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,438 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be18236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ab9835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:14:51,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:14:51,439 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:14:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:14:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:14:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322cfec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:14:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:14:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:14:51,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:14:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6ceb1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:14:51,442 DEBUG [Thread-1311 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,442 DEBUG [Thread-1298 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,442 DEBUG [Thread-1313 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:51,442 DEBUG [Thread-1302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,442 DEBUG [Thread-1306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-06T06:14:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,443 DEBUG [Thread-1315 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T06:14:51,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,445 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:51,446 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:51,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,447 DEBUG [Thread-1300 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:51,447 DEBUG [Thread-1317 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,448 DEBUG [Thread-1304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,449 DEBUG [Thread-1317 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,449 DEBUG [Thread-1317 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,450 DEBUG [Thread-1309 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:14:51,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,451 DEBUG [Thread-1309 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,452 DEBUG [Thread-1309 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:14:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:14:51,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:14:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:51,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:14:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:14:51,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/27a3339547044648baad2fe550682b34 is 50, key is test_row_0/A:col10/1733465691455/Put/seqid=0 2024-12-06T06:14:51,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465691481, max timestamp=1733465691491Num uniques 859; 2024-12-06T06:14:51,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465691481, max timestamp=1733465691491Num uniques 859; 2024-12-06T06:14:51,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:51,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:14:51,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465691481, max timestamp=1733465691493Num uniques 859; 2024-12-06T06:14:51,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:14:51,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742110_1286 (size=12001) 2024-12-06T06:14:51,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/27a3339547044648baad2fe550682b34 2024-12-06T06:14:51,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/14dd9ec2988841f29cd7965c4cb6dca4 is 50, key is test_row_0/B:col10/1733465691455/Put/seqid=0 2024-12-06T06:14:51,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742111_1287 (size=12001) 2024-12-06T06:14:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T06:14:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465751565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465751566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465751566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465751568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465751574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T06:14:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:51,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465751683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465751683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465751683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465751684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465751694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T06:14:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:51,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:51,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T06:14:51,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465751894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465751894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465751894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465751895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:51,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T06:14:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465751904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:51,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:51,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:51,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:51,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/14dd9ec2988841f29cd7965c4cb6dca4 2024-12-06T06:14:51,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/a3806ac44be14c7f8c514f7fd3a5c9f7 is 50, key is test_row_0/C:col10/1733465691455/Put/seqid=0 2024-12-06T06:14:51,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742112_1288 (size=12001) 2024-12-06T06:14:51,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/a3806ac44be14c7f8c514f7fd3a5c9f7 2024-12-06T06:14:51,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/27a3339547044648baad2fe550682b34 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/27a3339547044648baad2fe550682b34 2024-12-06T06:14:52,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/27a3339547044648baad2fe550682b34, entries=150, sequenceid=11, filesize=11.7 K 2024-12-06T06:14:52,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/14dd9ec2988841f29cd7965c4cb6dca4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/14dd9ec2988841f29cd7965c4cb6dca4 2024-12-06T06:14:52,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/14dd9ec2988841f29cd7965c4cb6dca4, entries=150, sequenceid=11, filesize=11.7 K 2024-12-06T06:14:52,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/a3806ac44be14c7f8c514f7fd3a5c9f7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a3806ac44be14c7f8c514f7fd3a5c9f7 2024-12-06T06:14:52,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a3806ac44be14c7f8c514f7fd3a5c9f7, entries=150, sequenceid=11, filesize=11.7 K 2024-12-06T06:14:52,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for aa1db8fd546545da53b24cabfd6bfcf1 in 557ms, sequenceid=11, compaction requested=false 2024-12-06T06:14:52,014 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T06:14:52,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:52,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:52,060 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-12-06T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:14:52,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/df7140dbd7fd422c8f949bac4788f175 is 50, key is test_row_0/A:col10/1733465691564/Put/seqid=0 2024-12-06T06:14:52,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742113_1289 (size=12001) 2024-12-06T06:14:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T06:14:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:52,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:52,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465752237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465752238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465752242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465752243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465752244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465752353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465752353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465752353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465752353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465752354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,472 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/df7140dbd7fd422c8f949bac4788f175 2024-12-06T06:14:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/bf749691e4004a7880db59fc219dbcdb is 50, key is test_row_0/B:col10/1733465691564/Put/seqid=0 2024-12-06T06:14:52,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742114_1290 (size=12001) 2024-12-06T06:14:52,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465752563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465752563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465752563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465752564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465752564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T06:14:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465752874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465752874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465752874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465752874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465752874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:52,884 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/bf749691e4004a7880db59fc219dbcdb 2024-12-06T06:14:52,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/cb7bebec57fc43ca804cdc3d24b14b91 is 50, key is test_row_0/C:col10/1733465691564/Put/seqid=0 2024-12-06T06:14:52,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742115_1291 (size=12001) 2024-12-06T06:14:52,904 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/cb7bebec57fc43ca804cdc3d24b14b91 2024-12-06T06:14:52,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/df7140dbd7fd422c8f949bac4788f175 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7140dbd7fd422c8f949bac4788f175 2024-12-06T06:14:52,922 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7140dbd7fd422c8f949bac4788f175, entries=150, sequenceid=47, filesize=11.7 K 2024-12-06T06:14:52,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/bf749691e4004a7880db59fc219dbcdb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bf749691e4004a7880db59fc219dbcdb 2024-12-06T06:14:52,943 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bf749691e4004a7880db59fc219dbcdb, entries=150, sequenceid=47, filesize=11.7 K 2024-12-06T06:14:52,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/cb7bebec57fc43ca804cdc3d24b14b91 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/cb7bebec57fc43ca804cdc3d24b14b91 2024-12-06T06:14:52,957 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/cb7bebec57fc43ca804cdc3d24b14b91, entries=150, sequenceid=47, filesize=11.7 K 2024-12-06T06:14:52,958 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for aa1db8fd546545da53b24cabfd6bfcf1 in 898ms, sequenceid=47, compaction requested=false 2024-12-06T06:14:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:52,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:52,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-06T06:14:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-06T06:14:52,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-06T06:14:52,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5140 sec 2024-12-06T06:14:52,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5230 sec 2024-12-06T06:14:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:53,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T06:14:53,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:53,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:53,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:53,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:53,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:53,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:53,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/e10039a6daee454e95895f28b1d34e67 is 50, key is test_row_0/A:col10/1733465693386/Put/seqid=0 2024-12-06T06:14:53,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742116_1292 (size=12001) 2024-12-06T06:14:53,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/e10039a6daee454e95895f28b1d34e67 2024-12-06T06:14:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465753412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465753414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465753417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/d78f1c6d0bc7450ab436f19b7d7cbcf0 is 50, key is test_row_0/B:col10/1733465693386/Put/seqid=0 2024-12-06T06:14:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465753418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465753423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742117_1293 (size=12001) 2024-12-06T06:14:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465753524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465753533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465753533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465753534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465753534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T06:14:53,594 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:53,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-06T06:14:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T06:14:53,597 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:53,599 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:53,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:53,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T06:14:53,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465753734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465753743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465753743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465753743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465753744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:53,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T06:14:53,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:53,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:53,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:53,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:53,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:53,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/d78f1c6d0bc7450ab436f19b7d7cbcf0 2024-12-06T06:14:53,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/bad29907c76242e9b9884c31d4b6af4d is 50, key is test_row_0/C:col10/1733465693386/Put/seqid=0 2024-12-06T06:14:53,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742118_1294 (size=12001) 2024-12-06T06:14:53,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T06:14:53,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:53,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:53,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:53,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:53,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T06:14:54,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465754044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465754053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465754054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465754054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465754054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T06:14:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:54,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T06:14:54,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:54,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:54,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:54,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:54,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T06:14:54,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/bad29907c76242e9b9884c31d4b6af4d 2024-12-06T06:14:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/e10039a6daee454e95895f28b1d34e67 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/e10039a6daee454e95895f28b1d34e67 2024-12-06T06:14:54,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/e10039a6daee454e95895f28b1d34e67, entries=150, sequenceid=60, filesize=11.7 K 2024-12-06T06:14:54,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/d78f1c6d0bc7450ab436f19b7d7cbcf0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d78f1c6d0bc7450ab436f19b7d7cbcf0 2024-12-06T06:14:54,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d78f1c6d0bc7450ab436f19b7d7cbcf0, entries=150, sequenceid=60, filesize=11.7 K 2024-12-06T06:14:54,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/bad29907c76242e9b9884c31d4b6af4d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/bad29907c76242e9b9884c31d4b6af4d 2024-12-06T06:14:54,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/bad29907c76242e9b9884c31d4b6af4d, entries=150, sequenceid=60, filesize=11.7 K 2024-12-06T06:14:54,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aa1db8fd546545da53b24cabfd6bfcf1 in 916ms, sequenceid=60, compaction requested=true 2024-12-06T06:14:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:54,305 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:54,305 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:54,306 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:54,306 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:54,306 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:14:54,306 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:14:54,306 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:54,306 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:54,307 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/14dd9ec2988841f29cd7965c4cb6dca4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bf749691e4004a7880db59fc219dbcdb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d78f1c6d0bc7450ab436f19b7d7cbcf0] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.2 K 2024-12-06T06:14:54,307 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/27a3339547044648baad2fe550682b34, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7140dbd7fd422c8f949bac4788f175, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/e10039a6daee454e95895f28b1d34e67] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.2 K 2024-12-06T06:14:54,307 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 14dd9ec2988841f29cd7965c4cb6dca4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733465691455 2024-12-06T06:14:54,307 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27a3339547044648baad2fe550682b34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733465691455 2024-12-06T06:14:54,307 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting df7140dbd7fd422c8f949bac4788f175, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733465691515 2024-12-06T06:14:54,307 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting bf749691e4004a7880db59fc219dbcdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733465691515 2024-12-06T06:14:54,308 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d78f1c6d0bc7450ab436f19b7d7cbcf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465692207 2024-12-06T06:14:54,308 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e10039a6daee454e95895f28b1d34e67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465692207 2024-12-06T06:14:54,316 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:54,316 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:54,316 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/6fc7c7b51d414bd0b26c382d08c47006 is 50, key is test_row_0/B:col10/1733465693386/Put/seqid=0 2024-12-06T06:14:54,316 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/ffd555fbd5ff47c08f722fde74de28cc is 50, key is test_row_0/A:col10/1733465693386/Put/seqid=0 2024-12-06T06:14:54,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742120_1296 (size=12104) 2024-12-06T06:14:54,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742119_1295 (size=12104) 2024-12-06T06:14:54,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T06:14:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:54,377 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T06:14:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/b51940ac1b324b2baf78de7f63c51ba8 is 50, key is test_row_0/A:col10/1733465693416/Put/seqid=0 2024-12-06T06:14:54,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742121_1297 (size=12001) 2024-12-06T06:14:54,418 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T06:14:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:54,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:54,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465754570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465754573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465754574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465754575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465754576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465754683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465754683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465754684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465754694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,727 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/ffd555fbd5ff47c08f722fde74de28cc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/ffd555fbd5ff47c08f722fde74de28cc 2024-12-06T06:14:54,730 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/6fc7c7b51d414bd0b26c382d08c47006 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6fc7c7b51d414bd0b26c382d08c47006 2024-12-06T06:14:54,733 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into ffd555fbd5ff47c08f722fde74de28cc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:54,733 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:54,733 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465694305; duration=0sec 2024-12-06T06:14:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T06:14:54,733 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:54,734 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:14:54,734 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:54,735 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:54,735 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:14:54,736 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:54,736 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a3806ac44be14c7f8c514f7fd3a5c9f7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/cb7bebec57fc43ca804cdc3d24b14b91, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/bad29907c76242e9b9884c31d4b6af4d] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.2 K 2024-12-06T06:14:54,736 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 6fc7c7b51d414bd0b26c382d08c47006(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:54,736 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:54,736 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465694305; duration=0sec 2024-12-06T06:14:54,736 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:54,736 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:14:54,736 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3806ac44be14c7f8c514f7fd3a5c9f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733465691455 2024-12-06T06:14:54,737 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb7bebec57fc43ca804cdc3d24b14b91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733465691515 2024-12-06T06:14:54,737 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad29907c76242e9b9884c31d4b6af4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465692207 2024-12-06T06:14:54,743 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:54,743 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/749dcd5011174ef6842601fc0ff2bfc8 is 50, key is test_row_0/C:col10/1733465693386/Put/seqid=0 2024-12-06T06:14:54,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742122_1298 (size=12104) 2024-12-06T06:14:54,788 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/b51940ac1b324b2baf78de7f63c51ba8 2024-12-06T06:14:54,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/1ec346784cf04cab9bcd6ff837d81ed6 is 50, key is test_row_0/B:col10/1733465693416/Put/seqid=0 2024-12-06T06:14:54,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742123_1299 (size=12001) 2024-12-06T06:14:54,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465754893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465754894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465754894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:54,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465754904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,152 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/749dcd5011174ef6842601fc0ff2bfc8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/749dcd5011174ef6842601fc0ff2bfc8 2024-12-06T06:14:55,157 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into 749dcd5011174ef6842601fc0ff2bfc8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:55,157 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:55,157 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465694305; duration=0sec 2024-12-06T06:14:55,157 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:55,157 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:14:55,201 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/1ec346784cf04cab9bcd6ff837d81ed6 2024-12-06T06:14:55,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465755204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465755204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465755204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/636840599cdd4a868f52a5cc5f394361 is 50, key is test_row_0/C:col10/1733465693416/Put/seqid=0 2024-12-06T06:14:55,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742124_1300 (size=12001) 2024-12-06T06:14:55,215 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/636840599cdd4a868f52a5cc5f394361 2024-12-06T06:14:55,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465755214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/b51940ac1b324b2baf78de7f63c51ba8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b51940ac1b324b2baf78de7f63c51ba8 2024-12-06T06:14:55,225 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b51940ac1b324b2baf78de7f63c51ba8, entries=150, sequenceid=79, filesize=11.7 K 2024-12-06T06:14:55,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/1ec346784cf04cab9bcd6ff837d81ed6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1ec346784cf04cab9bcd6ff837d81ed6 2024-12-06T06:14:55,231 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1ec346784cf04cab9bcd6ff837d81ed6, entries=150, sequenceid=79, filesize=11.7 K 2024-12-06T06:14:55,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/636840599cdd4a868f52a5cc5f394361 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/636840599cdd4a868f52a5cc5f394361 2024-12-06T06:14:55,237 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/636840599cdd4a868f52a5cc5f394361, entries=150, sequenceid=79, filesize=11.7 K 2024-12-06T06:14:55,237 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for aa1db8fd546545da53b24cabfd6bfcf1 in 860ms, sequenceid=79, compaction requested=false 2024-12-06T06:14:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-06T06:14:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-06T06:14:55,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-06T06:14:55,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6400 sec 2024-12-06T06:14:55,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6480 sec 2024-12-06T06:14:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:55,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T06:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:55,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/18a687b4e98d4d3abfd4ce3ac757707a is 50, key is test_row_0/A:col10/1733465695585/Put/seqid=0 2024-12-06T06:14:55,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742125_1301 (size=14341) 2024-12-06T06:14:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465755680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465755714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465755714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465755714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465755723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T06:14:55,744 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:55,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-06T06:14:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T06:14:55,746 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:55,747 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:55,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:55,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465755793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T06:14:55,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-06T06:14:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:55,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:56,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465756004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:56,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/18a687b4e98d4d3abfd4ce3ac757707a 2024-12-06T06:14:56,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/7fb5cb81e3b34a939bf3e7e988790a00 is 50, key is test_row_0/B:col10/1733465695585/Put/seqid=0 2024-12-06T06:14:56,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742126_1302 (size=12001) 2024-12-06T06:14:56,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/7fb5cb81e3b34a939bf3e7e988790a00 2024-12-06T06:14:56,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-06T06:14:56,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:56,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:56,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:56,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:56,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:56,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/2770e174a9404e6fba768f1f50d383d0 is 50, key is test_row_0/C:col10/1733465695585/Put/seqid=0 2024-12-06T06:14:56,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742127_1303 (size=12001) 2024-12-06T06:14:56,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/2770e174a9404e6fba768f1f50d383d0 2024-12-06T06:14:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T06:14:56,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/18a687b4e98d4d3abfd4ce3ac757707a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/18a687b4e98d4d3abfd4ce3ac757707a 2024-12-06T06:14:56,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/18a687b4e98d4d3abfd4ce3ac757707a, entries=200, sequenceid=96, filesize=14.0 K 2024-12-06T06:14:56,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/7fb5cb81e3b34a939bf3e7e988790a00 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/7fb5cb81e3b34a939bf3e7e988790a00 2024-12-06T06:14:56,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/7fb5cb81e3b34a939bf3e7e988790a00, entries=150, sequenceid=96, filesize=11.7 K 2024-12-06T06:14:56,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/2770e174a9404e6fba768f1f50d383d0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/2770e174a9404e6fba768f1f50d383d0 2024-12-06T06:14:56,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/2770e174a9404e6fba768f1f50d383d0, entries=150, sequenceid=96, filesize=11.7 K 2024-12-06T06:14:56,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for aa1db8fd546545da53b24cabfd6bfcf1 in 493ms, sequenceid=96, compaction requested=true 2024-12-06T06:14:56,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:56,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:56,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:56,080 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:56,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:56,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:56,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:56,080 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:56,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:56,081 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:56,081 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:14:56,081 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:56,081 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:14:56,081 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:56,081 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:56,081 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/ffd555fbd5ff47c08f722fde74de28cc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b51940ac1b324b2baf78de7f63c51ba8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/18a687b4e98d4d3abfd4ce3ac757707a] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=37.5 K 2024-12-06T06:14:56,081 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6fc7c7b51d414bd0b26c382d08c47006, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1ec346784cf04cab9bcd6ff837d81ed6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/7fb5cb81e3b34a939bf3e7e988790a00] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.3 K 2024-12-06T06:14:56,081 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd555fbd5ff47c08f722fde74de28cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465692207 2024-12-06T06:14:56,081 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fc7c7b51d414bd0b26c382d08c47006, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465692207 2024-12-06T06:14:56,082 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51940ac1b324b2baf78de7f63c51ba8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733465693409 2024-12-06T06:14:56,082 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec346784cf04cab9bcd6ff837d81ed6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733465693409 2024-12-06T06:14:56,082 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fb5cb81e3b34a939bf3e7e988790a00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733465694572 2024-12-06T06:14:56,082 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18a687b4e98d4d3abfd4ce3ac757707a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733465694564 2024-12-06T06:14:56,091 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:56,092 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/49e30c8b60824c8f9a62cc6b12acf307 is 50, key is test_row_0/B:col10/1733465695585/Put/seqid=0 2024-12-06T06:14:56,095 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:56,095 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/513418c8568145a7950e388b4f099462 is 50, key is test_row_0/A:col10/1733465695585/Put/seqid=0 2024-12-06T06:14:56,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742129_1305 (size=12207) 2024-12-06T06:14:56,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742128_1304 (size=12207) 2024-12-06T06:14:56,107 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/513418c8568145a7950e388b4f099462 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/513418c8568145a7950e388b4f099462 2024-12-06T06:14:56,111 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into 513418c8568145a7950e388b4f099462(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:56,111 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:56,112 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465696080; duration=0sec 2024-12-06T06:14:56,112 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:56,112 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:14:56,112 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:56,113 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:56,113 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:14:56,113 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:56,113 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/749dcd5011174ef6842601fc0ff2bfc8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/636840599cdd4a868f52a5cc5f394361, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/2770e174a9404e6fba768f1f50d383d0] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.3 K 2024-12-06T06:14:56,113 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 749dcd5011174ef6842601fc0ff2bfc8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465692207 2024-12-06T06:14:56,114 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 636840599cdd4a868f52a5cc5f394361, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733465693409 2024-12-06T06:14:56,114 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2770e174a9404e6fba768f1f50d383d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733465694572 2024-12-06T06:14:56,121 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:56,122 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/ee78d4ea18724de3a290f886c2202f43 is 50, key is test_row_0/C:col10/1733465695585/Put/seqid=0 2024-12-06T06:14:56,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742130_1306 (size=12207) 2024-12-06T06:14:56,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-06T06:14:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:56,207 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:14:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/d1b29741b58841a3a69757171c4abca2 is 50, key is test_row_0/A:col10/1733465695634/Put/seqid=0 2024-12-06T06:14:56,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742131_1307 (size=12001) 2024-12-06T06:14:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T06:14:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465756371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:56,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465756484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:56,506 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/49e30c8b60824c8f9a62cc6b12acf307 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49e30c8b60824c8f9a62cc6b12acf307 2024-12-06T06:14:56,511 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 49e30c8b60824c8f9a62cc6b12acf307(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:56,511 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:56,511 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465696080; duration=0sec 2024-12-06T06:14:56,511 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:56,511 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:14:56,531 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/ee78d4ea18724de3a290f886c2202f43 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ee78d4ea18724de3a290f886c2202f43 2024-12-06T06:14:56,535 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into ee78d4ea18724de3a290f886c2202f43(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:56,535 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:56,535 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465696080; duration=0sec 2024-12-06T06:14:56,535 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:56,535 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:14:56,618 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/d1b29741b58841a3a69757171c4abca2 2024-12-06T06:14:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/1cb6c87cff6f4e37ae1510b3a88d8a99 is 50, key is test_row_0/B:col10/1733465695634/Put/seqid=0 2024-12-06T06:14:56,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742132_1308 (size=12001) 2024-12-06T06:14:56,629 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/1cb6c87cff6f4e37ae1510b3a88d8a99 2024-12-06T06:14:56,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/9508f3f1135244b78bfc1cc762ca82c1 is 50, key is test_row_0/C:col10/1733465695634/Put/seqid=0 2024-12-06T06:14:56,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742133_1309 (size=12001) 2024-12-06T06:14:56,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465756693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:56,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465756724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:56,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465756724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:56,726 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:56,725 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:56,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465756733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:56,737 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:56,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465756743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:56,746 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T06:14:57,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465757004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:57,042 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/9508f3f1135244b78bfc1cc762ca82c1 2024-12-06T06:14:57,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/d1b29741b58841a3a69757171c4abca2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d1b29741b58841a3a69757171c4abca2 2024-12-06T06:14:57,050 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d1b29741b58841a3a69757171c4abca2, entries=150, sequenceid=115, filesize=11.7 K 2024-12-06T06:14:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/1cb6c87cff6f4e37ae1510b3a88d8a99 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1cb6c87cff6f4e37ae1510b3a88d8a99 2024-12-06T06:14:57,055 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1cb6c87cff6f4e37ae1510b3a88d8a99, entries=150, sequenceid=115, filesize=11.7 K 2024-12-06T06:14:57,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/9508f3f1135244b78bfc1cc762ca82c1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/9508f3f1135244b78bfc1cc762ca82c1 2024-12-06T06:14:57,060 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/9508f3f1135244b78bfc1cc762ca82c1, entries=150, sequenceid=115, filesize=11.7 K 2024-12-06T06:14:57,061 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for aa1db8fd546545da53b24cabfd6bfcf1 in 854ms, sequenceid=115, compaction requested=false 2024-12-06T06:14:57,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:57,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:57,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-06T06:14:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-06T06:14:57,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-06T06:14:57,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3150 sec 2024-12-06T06:14:57,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.3200 sec 2024-12-06T06:14:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:57,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:57,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/29d76126789c47c881ae42f1d42a7801 is 50, key is test_row_0/A:col10/1733465696347/Put/seqid=0 2024-12-06T06:14:57,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742134_1310 (size=14391) 2024-12-06T06:14:57,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465757604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:57,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465757714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T06:14:57,894 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:14:57,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:14:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-06T06:14:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T06:14:57,902 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:14:57,903 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:14:57,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:14:57,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465757923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:57,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/29d76126789c47c881ae42f1d42a7801 2024-12-06T06:14:57,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/eeae7c5f82b343da9d56a5691cdfc95c is 50, key is test_row_0/B:col10/1733465696347/Put/seqid=0 2024-12-06T06:14:57,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742135_1311 (size=12051) 2024-12-06T06:14:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T06:14:58,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T06:14:58,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:58,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T06:14:58,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:58,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T06:14:58,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465758233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/eeae7c5f82b343da9d56a5691cdfc95c 2024-12-06T06:14:58,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/ff8d9450c2a24eafb0e8c5c07fe496cd is 50, key is test_row_0/C:col10/1733465696347/Put/seqid=0 2024-12-06T06:14:58,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T06:14:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:14:58,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742136_1312 (size=12051) 2024-12-06T06:14:58,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/ff8d9450c2a24eafb0e8c5c07fe496cd 2024-12-06T06:14:58,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/29d76126789c47c881ae42f1d42a7801 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/29d76126789c47c881ae42f1d42a7801 2024-12-06T06:14:58,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/29d76126789c47c881ae42f1d42a7801, entries=200, sequenceid=132, filesize=14.1 K 2024-12-06T06:14:58,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/eeae7c5f82b343da9d56a5691cdfc95c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eeae7c5f82b343da9d56a5691cdfc95c 2024-12-06T06:14:58,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eeae7c5f82b343da9d56a5691cdfc95c, entries=150, sequenceid=132, filesize=11.8 K 2024-12-06T06:14:58,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/ff8d9450c2a24eafb0e8c5c07fe496cd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ff8d9450c2a24eafb0e8c5c07fe496cd 2024-12-06T06:14:58,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ff8d9450c2a24eafb0e8c5c07fe496cd, entries=150, sequenceid=132, filesize=11.8 K 2024-12-06T06:14:58,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for aa1db8fd546545da53b24cabfd6bfcf1 in 896ms, sequenceid=132, compaction requested=true 2024-12-06T06:14:58,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:58,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:14:58,414 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:58,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:58,414 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:58,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:14:58,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:58,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:14:58,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:58,416 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:58,416 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:14:58,416 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,416 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/513418c8568145a7950e388b4f099462, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d1b29741b58841a3a69757171c4abca2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/29d76126789c47c881ae42f1d42a7801] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=37.7 K 2024-12-06T06:14:58,416 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:58,416 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:14:58,416 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,416 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49e30c8b60824c8f9a62cc6b12acf307, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1cb6c87cff6f4e37ae1510b3a88d8a99, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eeae7c5f82b343da9d56a5691cdfc95c] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.4 K 2024-12-06T06:14:58,417 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 49e30c8b60824c8f9a62cc6b12acf307, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733465694572 2024-12-06T06:14:58,417 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 513418c8568145a7950e388b4f099462, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733465694572 2024-12-06T06:14:58,417 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb6c87cff6f4e37ae1510b3a88d8a99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465695634 2024-12-06T06:14:58,417 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b29741b58841a3a69757171c4abca2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465695634 2024-12-06T06:14:58,418 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d76126789c47c881ae42f1d42a7801, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733465696335 2024-12-06T06:14:58,418 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting eeae7c5f82b343da9d56a5691cdfc95c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733465696347 2024-12-06T06:14:58,455 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:58,460 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/8304144f81b147d8b190846ab7abd473 is 50, key is test_row_0/B:col10/1733465696347/Put/seqid=0 2024-12-06T06:14:58,462 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:58,462 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/b4be0219c2b44cf3a3a97d206cc92123 is 50, key is test_row_0/A:col10/1733465696347/Put/seqid=0 2024-12-06T06:14:58,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742137_1313 (size=12359) 2024-12-06T06:14:58,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742138_1314 (size=12359) 2024-12-06T06:14:58,505 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/8304144f81b147d8b190846ab7abd473 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8304144f81b147d8b190846ab7abd473 2024-12-06T06:14:58,511 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/b4be0219c2b44cf3a3a97d206cc92123 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b4be0219c2b44cf3a3a97d206cc92123 2024-12-06T06:14:58,515 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 8304144f81b147d8b190846ab7abd473(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:58,515 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:58,515 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465698414; duration=0sec 2024-12-06T06:14:58,515 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:14:58,515 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:14:58,515 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:14:58,517 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into b4be0219c2b44cf3a3a97d206cc92123(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:58,517 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:58,518 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465698413; duration=0sec 2024-12-06T06:14:58,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-06T06:14:58,518 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:58,518 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,518 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T06:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:58,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:58,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:58,519 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:14:58,519 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:14:58,519 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:58,519 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ee78d4ea18724de3a290f886c2202f43, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/9508f3f1135244b78bfc1cc762ca82c1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ff8d9450c2a24eafb0e8c5c07fe496cd] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.4 K 2024-12-06T06:14:58,519 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ee78d4ea18724de3a290f886c2202f43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733465694572 2024-12-06T06:14:58,520 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9508f3f1135244b78bfc1cc762ca82c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465695634 2024-12-06T06:14:58,521 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8d9450c2a24eafb0e8c5c07fe496cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733465696347 2024-12-06T06:14:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/952e480a842f481592f337dcd94a496c is 50, key is test_row_0/A:col10/1733465697538/Put/seqid=0 2024-12-06T06:14:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T06:14:58,539 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:14:58,539 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/44159779f56942d7b540013ed42b1584 is 50, key is test_row_0/C:col10/1733465696347/Put/seqid=0 2024-12-06T06:14:58,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742139_1315 (size=12151) 2024-12-06T06:14:58,547 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/952e480a842f481592f337dcd94a496c 2024-12-06T06:14:58,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/eb6a51bd9d9d4e10a4317b864f8c3ebc is 50, key is test_row_0/B:col10/1733465697538/Put/seqid=0 2024-12-06T06:14:58,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742140_1316 (size=12359) 2024-12-06T06:14:58,578 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/44159779f56942d7b540013ed42b1584 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/44159779f56942d7b540013ed42b1584 2024-12-06T06:14:58,585 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into 44159779f56942d7b540013ed42b1584(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:14:58,585 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:58,585 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465698414; duration=0sec 2024-12-06T06:14:58,585 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:14:58,585 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:14:58,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742141_1317 (size=12151) 2024-12-06T06:14:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:58,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:14:58,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465758761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465758764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,768 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:14:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465758766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465758768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465758769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465758873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465758884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465758884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465758884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:58,987 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/eb6a51bd9d9d4e10a4317b864f8c3ebc 2024-12-06T06:14:58,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/771724ae62a840158646fb920791889a is 50, key is test_row_0/C:col10/1733465697538/Put/seqid=0 2024-12-06T06:14:59,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742142_1318 (size=12151) 2024-12-06T06:14:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T06:14:59,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465759083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465759094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465759094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465759094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465759393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,401 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/771724ae62a840158646fb920791889a 2024-12-06T06:14:59,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/952e480a842f481592f337dcd94a496c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/952e480a842f481592f337dcd94a496c 2024-12-06T06:14:59,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465759404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465759404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465759404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,409 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/952e480a842f481592f337dcd94a496c, entries=150, sequenceid=151, filesize=11.9 K 2024-12-06T06:14:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/eb6a51bd9d9d4e10a4317b864f8c3ebc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eb6a51bd9d9d4e10a4317b864f8c3ebc 2024-12-06T06:14:59,416 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eb6a51bd9d9d4e10a4317b864f8c3ebc, entries=150, sequenceid=151, filesize=11.9 K 2024-12-06T06:14:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/771724ae62a840158646fb920791889a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/771724ae62a840158646fb920791889a 2024-12-06T06:14:59,421 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/771724ae62a840158646fb920791889a, entries=150, sequenceid=151, filesize=11.9 K 2024-12-06T06:14:59,422 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for aa1db8fd546545da53b24cabfd6bfcf1 in 904ms, sequenceid=151, compaction requested=false 2024-12-06T06:14:59,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:14:59,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:14:59,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-06T06:14:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-06T06:14:59,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-06T06:14:59,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5200 sec 2024-12-06T06:14:59,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5310 sec 2024-12-06T06:14:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:14:59,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T06:14:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:14:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:14:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:14:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:14:59,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/3bbc8f0bdf05416cb9afba8a76f4fd73 is 50, key is test_row_0/A:col10/1733465698757/Put/seqid=0 2024-12-06T06:14:59,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742143_1319 (size=12151) 2024-12-06T06:14:59,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/3bbc8f0bdf05416cb9afba8a76f4fd73 2024-12-06T06:14:59,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/80a3063ac0e245e1af13cd86c1e8d5b5 is 50, key is test_row_0/B:col10/1733465698757/Put/seqid=0 2024-12-06T06:14:59,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465759933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465759935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465759937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:14:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465759937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:14:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:14:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:14:59,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742144_1320 (size=12151) 2024-12-06T06:15:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T06:15:00,045 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:00,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-06T06:15:00,047 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T06:15:00,047 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:00,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:00,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465760044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465760044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465760054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465760054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T06:15:00,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-06T06:15:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465760254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465760254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465760264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465760264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-06T06:15:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/80a3063ac0e245e1af13cd86c1e8d5b5 2024-12-06T06:15:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T06:15:00,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/6d0100ebbed644aca7752b0ac3d9cb96 is 50, key is test_row_0/C:col10/1733465698757/Put/seqid=0 2024-12-06T06:15:00,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742145_1321 (size=12151) 2024-12-06T06:15:00,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/6d0100ebbed644aca7752b0ac3d9cb96 2024-12-06T06:15:00,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/3bbc8f0bdf05416cb9afba8a76f4fd73 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3bbc8f0bdf05416cb9afba8a76f4fd73 2024-12-06T06:15:00,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3bbc8f0bdf05416cb9afba8a76f4fd73, entries=150, sequenceid=169, filesize=11.9 K 2024-12-06T06:15:00,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/80a3063ac0e245e1af13cd86c1e8d5b5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/80a3063ac0e245e1af13cd86c1e8d5b5 2024-12-06T06:15:00,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/80a3063ac0e245e1af13cd86c1e8d5b5, entries=150, sequenceid=169, filesize=11.9 K 2024-12-06T06:15:00,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/6d0100ebbed644aca7752b0ac3d9cb96 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6d0100ebbed644aca7752b0ac3d9cb96 2024-12-06T06:15:00,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6d0100ebbed644aca7752b0ac3d9cb96, entries=150, sequenceid=169, filesize=11.9 K 2024-12-06T06:15:00,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for aa1db8fd546545da53b24cabfd6bfcf1 in 495ms, sequenceid=169, compaction requested=true 2024-12-06T06:15:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:00,400 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:00,400 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:00,401 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:00,401 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:00,401 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:15:00,401 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:15:00,401 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,401 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,401 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8304144f81b147d8b190846ab7abd473, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eb6a51bd9d9d4e10a4317b864f8c3ebc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/80a3063ac0e245e1af13cd86c1e8d5b5] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.8 K 2024-12-06T06:15:00,401 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b4be0219c2b44cf3a3a97d206cc92123, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/952e480a842f481592f337dcd94a496c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3bbc8f0bdf05416cb9afba8a76f4fd73] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=35.8 K 2024-12-06T06:15:00,402 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4be0219c2b44cf3a3a97d206cc92123, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733465696347 2024-12-06T06:15:00,402 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8304144f81b147d8b190846ab7abd473, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733465696347 2024-12-06T06:15:00,404 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting eb6a51bd9d9d4e10a4317b864f8c3ebc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733465697538 2024-12-06T06:15:00,404 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 952e480a842f481592f337dcd94a496c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733465697538 2024-12-06T06:15:00,404 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a3063ac0e245e1af13cd86c1e8d5b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465698757 2024-12-06T06:15:00,404 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bbc8f0bdf05416cb9afba8a76f4fd73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465698757 2024-12-06T06:15:00,411 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:00,412 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/66c04aaf2ccc4936993278a8b3daade8 is 50, key is test_row_0/B:col10/1733465698757/Put/seqid=0 2024-12-06T06:15:00,415 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:00,415 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/3b3a03e3ad7241a1bb98d821d8d9c6c0 is 50, key is test_row_0/A:col10/1733465698757/Put/seqid=0 2024-12-06T06:15:00,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742146_1322 (size=12561) 2024-12-06T06:15:00,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742147_1323 (size=12561) 2024-12-06T06:15:00,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-06T06:15:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,507 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:15:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:00,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/3399ba691cb94349b3361e608608565a is 50, key is test_row_0/A:col10/1733465699935/Put/seqid=0 2024-12-06T06:15:00,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742148_1324 (size=12151) 2024-12-06T06:15:00,524 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/3399ba691cb94349b3361e608608565a 2024-12-06T06:15:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/16a372703a894603b423dbddce690c2d is 50, key is test_row_0/B:col10/1733465699935/Put/seqid=0 2024-12-06T06:15:00,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742149_1325 (size=12151) 2024-12-06T06:15:00,550 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/16a372703a894603b423dbddce690c2d 2024-12-06T06:15:00,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/49ab7923e38942e88d1e468b6537643a is 50, key is test_row_0/C:col10/1733465699935/Put/seqid=0 2024-12-06T06:15:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:00,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:00,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742150_1326 (size=12151) 2024-12-06T06:15:00,594 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/49ab7923e38942e88d1e468b6537643a 2024-12-06T06:15:00,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465760589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465760590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465760591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465760594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/3399ba691cb94349b3361e608608565a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3399ba691cb94349b3361e608608565a 2024-12-06T06:15:00,619 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3399ba691cb94349b3361e608608565a, entries=150, sequenceid=184, filesize=11.9 K 2024-12-06T06:15:00,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/16a372703a894603b423dbddce690c2d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/16a372703a894603b423dbddce690c2d 2024-12-06T06:15:00,625 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/16a372703a894603b423dbddce690c2d, entries=150, sequenceid=184, filesize=11.9 K 2024-12-06T06:15:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/49ab7923e38942e88d1e468b6537643a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/49ab7923e38942e88d1e468b6537643a 2024-12-06T06:15:00,630 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/49ab7923e38942e88d1e468b6537643a, entries=150, sequenceid=184, filesize=11.9 K 2024-12-06T06:15:00,631 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for aa1db8fd546545da53b24cabfd6bfcf1 in 124ms, sequenceid=184, compaction requested=true 2024-12-06T06:15:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-06T06:15:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-06T06:15:00,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-06T06:15:00,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 587 msec 2024-12-06T06:15:00,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 591 msec 2024-12-06T06:15:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-06T06:15:00,674 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:00,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-06T06:15:00,680 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T06:15:00,681 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:00,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:00,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:15:00,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:00,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:00,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:00,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:00,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:00,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:00,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/5b46c3e891e14c88b7ad8a30c217db34 is 50, key is test_row_0/A:col10/1733465700708/Put/seqid=0 2024-12-06T06:15:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465760733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465760736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465760736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465760738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T06:15:00,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742151_1327 (size=14541) 2024-12-06T06:15:00,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/5b46c3e891e14c88b7ad8a30c217db34 2024-12-06T06:15:00,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/b4bf4c8b61004d33a42929dde123237a is 50, key is test_row_0/B:col10/1733465700708/Put/seqid=0 2024-12-06T06:15:00,832 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/66c04aaf2ccc4936993278a8b3daade8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/66c04aaf2ccc4936993278a8b3daade8 2024-12-06T06:15:00,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T06:15:00,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:00,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,871 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 66c04aaf2ccc4936993278a8b3daade8(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:00,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:00,871 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465700400; duration=0sec 2024-12-06T06:15:00,871 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/3b3a03e3ad7241a1bb98d821d8d9c6c0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3b3a03e3ad7241a1bb98d821d8d9c6c0 2024-12-06T06:15:00,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:00,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:00,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:00,873 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:00,873 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:15:00,873 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,873 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/44159779f56942d7b540013ed42b1584, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/771724ae62a840158646fb920791889a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6d0100ebbed644aca7752b0ac3d9cb96, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/49ab7923e38942e88d1e468b6537643a] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=47.7 K 2024-12-06T06:15:00,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465760872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,874 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 44159779f56942d7b540013ed42b1584, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733465696347 2024-12-06T06:15:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465760872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,874 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 771724ae62a840158646fb920791889a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733465697538 2024-12-06T06:15:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,875 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0100ebbed644aca7752b0ac3d9cb96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465698757 2024-12-06T06:15:00,875 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 49ab7923e38942e88d1e468b6537643a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733465699923 2024-12-06T06:15:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465760872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465760872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:00,880 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into 3b3a03e3ad7241a1bb98d821d8d9c6c0(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:00,880 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:00,880 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465700400; duration=0sec 2024-12-06T06:15:00,880 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:00,880 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:00,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742152_1328 (size=12151) 2024-12-06T06:15:00,885 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:00,885 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/d7cf5230f0ae46c4be31aa9a96d54865 is 50, key is test_row_0/C:col10/1733465699935/Put/seqid=0 2024-12-06T06:15:00,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742153_1329 (size=12595) 2024-12-06T06:15:00,895 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/d7cf5230f0ae46c4be31aa9a96d54865 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d7cf5230f0ae46c4be31aa9a96d54865 2024-12-06T06:15:00,902 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into d7cf5230f0ae46c4be31aa9a96d54865(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:00,902 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:00,902 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=12, startTime=1733465700400; duration=0sec 2024-12-06T06:15:00,902 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:00,902 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:00,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T06:15:00,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:00,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:00,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T06:15:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465761084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465761085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465761085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465761085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T06:15:01,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:01,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/b4bf4c8b61004d33a42929dde123237a 2024-12-06T06:15:01,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/82856fd8c5c240f8842e726072cc1aa9 is 50, key is test_row_0/C:col10/1733465700708/Put/seqid=0 2024-12-06T06:15:01,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T06:15:01,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:01,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T06:15:01,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742154_1330 (size=12151) 2024-12-06T06:15:01,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/82856fd8c5c240f8842e726072cc1aa9 2024-12-06T06:15:01,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/5b46c3e891e14c88b7ad8a30c217db34 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/5b46c3e891e14c88b7ad8a30c217db34 2024-12-06T06:15:01,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/5b46c3e891e14c88b7ad8a30c217db34, entries=200, sequenceid=202, filesize=14.2 K 2024-12-06T06:15:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/b4bf4c8b61004d33a42929dde123237a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/b4bf4c8b61004d33a42929dde123237a 2024-12-06T06:15:01,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/b4bf4c8b61004d33a42929dde123237a, entries=150, sequenceid=202, filesize=11.9 K 2024-12-06T06:15:01,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/82856fd8c5c240f8842e726072cc1aa9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/82856fd8c5c240f8842e726072cc1aa9 2024-12-06T06:15:01,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/82856fd8c5c240f8842e726072cc1aa9, entries=150, sequenceid=202, filesize=11.9 K 2024-12-06T06:15:01,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for aa1db8fd546545da53b24cabfd6bfcf1 in 656ms, sequenceid=202, compaction requested=true 2024-12-06T06:15:01,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:01,366 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:01,367 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:01,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:01,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:01,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:01,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:01,368 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:01,368 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:15:01,368 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,369 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3b3a03e3ad7241a1bb98d821d8d9c6c0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3399ba691cb94349b3361e608608565a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/5b46c3e891e14c88b7ad8a30c217db34] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=38.3 K 2024-12-06T06:15:01,369 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:01,369 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:15:01,369 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,369 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/66c04aaf2ccc4936993278a8b3daade8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/16a372703a894603b423dbddce690c2d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/b4bf4c8b61004d33a42929dde123237a] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=36.0 K 2024-12-06T06:15:01,369 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b3a03e3ad7241a1bb98d821d8d9c6c0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465698757 2024-12-06T06:15:01,369 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 66c04aaf2ccc4936993278a8b3daade8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465698757 2024-12-06T06:15:01,370 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3399ba691cb94349b3361e608608565a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733465699923 2024-12-06T06:15:01,370 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a372703a894603b423dbddce690c2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733465699923 2024-12-06T06:15:01,370 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b46c3e891e14c88b7ad8a30c217db34, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733465700587 2024-12-06T06:15:01,370 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b4bf4c8b61004d33a42929dde123237a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733465700587 2024-12-06T06:15:01,386 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:01,387 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/d7f493435ff9469ca169eccf443f1acf is 50, key is test_row_0/B:col10/1733465700708/Put/seqid=0 2024-12-06T06:15:01,388 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:01,389 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/0c7d5846943d404d960f67230f8a130d is 50, key is test_row_0/A:col10/1733465700708/Put/seqid=0 2024-12-06T06:15:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:01,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:15:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:01,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742155_1331 (size=12663) 2024-12-06T06:15:01,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742156_1332 (size=12663) 2024-12-06T06:15:01,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/8cc451244fa64963bac12c14b831ce1f is 50, key is test_row_0/A:col10/1733465701399/Put/seqid=0 2024-12-06T06:15:01,412 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/d7f493435ff9469ca169eccf443f1acf as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d7f493435ff9469ca169eccf443f1acf 2024-12-06T06:15:01,417 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into d7f493435ff9469ca169eccf443f1acf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:01,418 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:01,418 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465701367; duration=0sec 2024-12-06T06:15:01,418 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:01,418 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:01,418 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-06T06:15:01,419 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-06T06:15:01,419 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-06T06:15:01,419 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. because compaction request was cancelled 2024-12-06T06:15:01,419 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:01,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742157_1333 (size=12151) 2024-12-06T06:15:01,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/8cc451244fa64963bac12c14b831ce1f 2024-12-06T06:15:01,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e398d35400f94343aa0a121f338edaa7 is 50, key is test_row_0/B:col10/1733465701399/Put/seqid=0 2024-12-06T06:15:01,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465761424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465761425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465761429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465761429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T06:15:01,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:01,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742158_1334 (size=12151) 2024-12-06T06:15:01,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e398d35400f94343aa0a121f338edaa7 2024-12-06T06:15:01,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/d33a30468c2a4ec8ac2e7ab42a76619a is 50, key is test_row_0/C:col10/1733465701399/Put/seqid=0 2024-12-06T06:15:01,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742159_1335 (size=12151) 2024-12-06T06:15:01,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/d33a30468c2a4ec8ac2e7ab42a76619a 2024-12-06T06:15:01,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/8cc451244fa64963bac12c14b831ce1f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/8cc451244fa64963bac12c14b831ce1f 2024-12-06T06:15:01,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/8cc451244fa64963bac12c14b831ce1f, entries=150, sequenceid=222, filesize=11.9 K 2024-12-06T06:15:01,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465761534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465761534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e398d35400f94343aa0a121f338edaa7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e398d35400f94343aa0a121f338edaa7 2024-12-06T06:15:01,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e398d35400f94343aa0a121f338edaa7, entries=150, sequenceid=222, filesize=11.9 K 2024-12-06T06:15:01,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/d33a30468c2a4ec8ac2e7ab42a76619a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d33a30468c2a4ec8ac2e7ab42a76619a 2024-12-06T06:15:01,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465761544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465761544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d33a30468c2a4ec8ac2e7ab42a76619a, entries=150, sequenceid=222, filesize=11.9 K 2024-12-06T06:15:01,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for aa1db8fd546545da53b24cabfd6bfcf1 in 154ms, sequenceid=222, compaction requested=true 2024-12-06T06:15:01,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:01,554 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-06T06:15:01,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:01,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:01,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:01,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:01,555 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-06T06:15:01,555 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-06T06:15:01,555 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. because compaction request was cancelled 2024-12-06T06:15:01,555 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:01,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:01,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:01,555 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:01,556 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:01,556 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:15:01,556 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,556 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d7cf5230f0ae46c4be31aa9a96d54865, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/82856fd8c5c240f8842e726072cc1aa9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d33a30468c2a4ec8ac2e7ab42a76619a] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=36.0 K 2024-12-06T06:15:01,557 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d7cf5230f0ae46c4be31aa9a96d54865, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733465699923 2024-12-06T06:15:01,557 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 82856fd8c5c240f8842e726072cc1aa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733465700587 2024-12-06T06:15:01,557 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d33a30468c2a4ec8ac2e7ab42a76619a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733465700733 2024-12-06T06:15:01,587 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:01,587 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/6dfad5bfb5da4b8a90c6988a0cd430f6 is 50, key is test_row_0/C:col10/1733465701399/Put/seqid=0 2024-12-06T06:15:01,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742160_1336 (size=12697) 2024-12-06T06:15:01,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-06T06:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:01,605 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:01,610 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/6dfad5bfb5da4b8a90c6988a0cd430f6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6dfad5bfb5da4b8a90c6988a0cd430f6 2024-12-06T06:15:01,615 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into 6dfad5bfb5da4b8a90c6988a0cd430f6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:01,615 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:01,615 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465701555; duration=0sec 2024-12-06T06:15:01,615 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:01,615 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:01,615 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-06T06:15:01,616 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-06T06:15:01,616 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-06T06:15:01,616 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. because compaction request was cancelled 2024-12-06T06:15:01,616 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/0fdc07222ac1485eb8fbd6b5910df231 is 50, key is test_row_0/A:col10/1733465701426/Put/seqid=0 2024-12-06T06:15:01,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742161_1337 (size=12151) 2024-12-06T06:15:01,644 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/0fdc07222ac1485eb8fbd6b5910df231 2024-12-06T06:15:01,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/610ab54b133540bcab14e170308efeb0 is 50, key is test_row_0/B:col10/1733465701426/Put/seqid=0 2024-12-06T06:15:01,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742162_1338 (size=12151) 2024-12-06T06:15:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:01,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465761776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465761777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465761781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465761783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T06:15:01,816 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/0c7d5846943d404d960f67230f8a130d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0c7d5846943d404d960f67230f8a130d 2024-12-06T06:15:01,824 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into 0c7d5846943d404d960f67230f8a130d(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:01,824 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:01,824 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465701366; duration=0sec 2024-12-06T06:15:01,824 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:01,824 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465761894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465761894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465761894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465761894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,076 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/610ab54b133540bcab14e170308efeb0 2024-12-06T06:15:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/62f2c81a221e423ca4fa0ed70bbe65d1 is 50, key is test_row_0/C:col10/1733465701426/Put/seqid=0 2024-12-06T06:15:02,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742163_1339 (size=12151) 2024-12-06T06:15:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465762103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465762104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465762104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465762104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465762418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465762418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465762418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465762418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,490 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/62f2c81a221e423ca4fa0ed70bbe65d1 2024-12-06T06:15:02,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/0fdc07222ac1485eb8fbd6b5910df231 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0fdc07222ac1485eb8fbd6b5910df231 2024-12-06T06:15:02,499 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0fdc07222ac1485eb8fbd6b5910df231, entries=150, sequenceid=238, filesize=11.9 K 2024-12-06T06:15:02,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/610ab54b133540bcab14e170308efeb0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/610ab54b133540bcab14e170308efeb0 2024-12-06T06:15:02,504 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/610ab54b133540bcab14e170308efeb0, entries=150, sequenceid=238, filesize=11.9 K 2024-12-06T06:15:02,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/62f2c81a221e423ca4fa0ed70bbe65d1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/62f2c81a221e423ca4fa0ed70bbe65d1 2024-12-06T06:15:02,509 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/62f2c81a221e423ca4fa0ed70bbe65d1, entries=150, sequenceid=238, filesize=11.9 K 2024-12-06T06:15:02,515 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for aa1db8fd546545da53b24cabfd6bfcf1 in 909ms, sequenceid=238, compaction requested=true 2024-12-06T06:15:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-06T06:15:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-06T06:15:02,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-06T06:15:02,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8360 sec 2024-12-06T06:15:02,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.8470 sec 2024-12-06T06:15:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:02,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:15:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:02,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/df7349db8e344e2ebfe9f9c305f5f615 is 50, key is test_row_0/A:col10/1733465701774/Put/seqid=0 2024-12-06T06:15:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T06:15:02,824 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:02,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-06T06:15:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T06:15:02,827 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:02,828 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:02,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:02,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742164_1340 (size=12201) 2024-12-06T06:15:02,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/df7349db8e344e2ebfe9f9c305f5f615 2024-12-06T06:15:02,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/bfcb62dc9fe0421bb4fe4eafe43d57d9 is 50, key is test_row_0/B:col10/1733465701774/Put/seqid=0 2024-12-06T06:15:02,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465762882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742165_1341 (size=12201) 2024-12-06T06:15:02,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/bfcb62dc9fe0421bb4fe4eafe43d57d9 2024-12-06T06:15:02,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/0849bdff0e6a40d6bf4969cf6e88c1c4 is 50, key is test_row_0/C:col10/1733465701774/Put/seqid=0 2024-12-06T06:15:02,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742166_1342 (size=12201) 2024-12-06T06:15:02,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/0849bdff0e6a40d6bf4969cf6e88c1c4 2024-12-06T06:15:02,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465762925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465762925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465762925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/df7349db8e344e2ebfe9f9c305f5f615 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7349db8e344e2ebfe9f9c305f5f615 2024-12-06T06:15:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T06:15:02,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465762934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:02,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7349db8e344e2ebfe9f9c305f5f615, entries=150, sequenceid=258, filesize=11.9 K 2024-12-06T06:15:02,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/bfcb62dc9fe0421bb4fe4eafe43d57d9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bfcb62dc9fe0421bb4fe4eafe43d57d9 2024-12-06T06:15:02,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bfcb62dc9fe0421bb4fe4eafe43d57d9, entries=150, sequenceid=258, filesize=11.9 K 2024-12-06T06:15:02,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/0849bdff0e6a40d6bf4969cf6e88c1c4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/0849bdff0e6a40d6bf4969cf6e88c1c4 2024-12-06T06:15:02,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/0849bdff0e6a40d6bf4969cf6e88c1c4, entries=150, sequenceid=258, filesize=11.9 K 2024-12-06T06:15:02,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for aa1db8fd546545da53b24cabfd6bfcf1 in 150ms, sequenceid=258, compaction requested=true 2024-12-06T06:15:02,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:02,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:02,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:02,953 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:02,953 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:02,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:02,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:02,957 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:02,957 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:02,957 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:15:02,957 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:02,957 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:15:02,957 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d7f493435ff9469ca169eccf443f1acf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e398d35400f94343aa0a121f338edaa7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/610ab54b133540bcab14e170308efeb0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bfcb62dc9fe0421bb4fe4eafe43d57d9] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=48.0 K 2024-12-06T06:15:02,957 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:02,957 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0c7d5846943d404d960f67230f8a130d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/8cc451244fa64963bac12c14b831ce1f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0fdc07222ac1485eb8fbd6b5910df231, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7349db8e344e2ebfe9f9c305f5f615] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=48.0 K 2024-12-06T06:15:02,958 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f493435ff9469ca169eccf443f1acf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733465700587 2024-12-06T06:15:02,958 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7d5846943d404d960f67230f8a130d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733465700587 2024-12-06T06:15:02,958 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc451244fa64963bac12c14b831ce1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733465700733 2024-12-06T06:15:02,959 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fdc07222ac1485eb8fbd6b5910df231, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733465701409 2024-12-06T06:15:02,959 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting df7349db8e344e2ebfe9f9c305f5f615, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733465701774 2024-12-06T06:15:02,959 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e398d35400f94343aa0a121f338edaa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733465700733 2024-12-06T06:15:02,960 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 610ab54b133540bcab14e170308efeb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733465701409 2024-12-06T06:15:02,960 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting bfcb62dc9fe0421bb4fe4eafe43d57d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733465701774 2024-12-06T06:15:02,972 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:02,972 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:02,973 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/6b48adbedb9f4f529ffd69d824054870 is 50, key is test_row_0/A:col10/1733465701774/Put/seqid=0 2024-12-06T06:15:02,973 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/2eaf680df6dc4e779cb3ed31d0041350 is 50, key is test_row_0/B:col10/1733465701774/Put/seqid=0 2024-12-06T06:15:02,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-06T06:15:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:02,981 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T06:15:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:02,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742167_1343 (size=12849) 2024-12-06T06:15:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:03,000 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/2eaf680df6dc4e779cb3ed31d0041350 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/2eaf680df6dc4e779cb3ed31d0041350 2024-12-06T06:15:03,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:03,006 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 2eaf680df6dc4e779cb3ed31d0041350(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:03,006 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:03,006 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=12, startTime=1733465702953; duration=0sec 2024-12-06T06:15:03,006 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:03,006 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:03,006 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:03,007 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:03,007 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:15:03,007 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:03,008 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6dfad5bfb5da4b8a90c6988a0cd430f6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/62f2c81a221e423ca4fa0ed70bbe65d1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/0849bdff0e6a40d6bf4969cf6e88c1c4] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=36.2 K 2024-12-06T06:15:03,009 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dfad5bfb5da4b8a90c6988a0cd430f6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733465700733 2024-12-06T06:15:03,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/fe885255287d49258c94a4982b053487 is 50, key is test_row_0/A:col10/1733465702853/Put/seqid=0 2024-12-06T06:15:03,011 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f2c81a221e423ca4fa0ed70bbe65d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733465701409 2024-12-06T06:15:03,012 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0849bdff0e6a40d6bf4969cf6e88c1c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733465701774 2024-12-06T06:15:03,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742168_1344 (size=12849) 2024-12-06T06:15:03,036 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:03,037 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/e5fdd4f7c237479994ad1e247471e3dd is 50, key is test_row_0/C:col10/1733465701774/Put/seqid=0 2024-12-06T06:15:03,037 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/6b48adbedb9f4f529ffd69d824054870 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6b48adbedb9f4f529ffd69d824054870 2024-12-06T06:15:03,046 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into 6b48adbedb9f4f529ffd69d824054870(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:03,046 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:03,046 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=12, startTime=1733465702953; duration=0sec 2024-12-06T06:15:03,046 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:03,046 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:03,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742169_1345 (size=12301) 2024-12-06T06:15:03,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742170_1346 (size=12849) 2024-12-06T06:15:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465763102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T06:15:03,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465763214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:03,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465763423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:03,447 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/fe885255287d49258c94a4982b053487 2024-12-06T06:15:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T06:15:03,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/a948356fc961442b8baaac314c0cd708 is 50, key is test_row_0/B:col10/1733465702853/Put/seqid=0 2024-12-06T06:15:03,471 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/e5fdd4f7c237479994ad1e247471e3dd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e5fdd4f7c237479994ad1e247471e3dd 2024-12-06T06:15:03,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742171_1347 (size=12301) 2024-12-06T06:15:03,486 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into e5fdd4f7c237479994ad1e247471e3dd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:03,486 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:03,486 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465702954; duration=0sec 2024-12-06T06:15:03,486 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:03,486 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:03,489 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/a948356fc961442b8baaac314c0cd708 2024-12-06T06:15:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/b9c395ed9d674d628a79f4ab37444852 is 50, key is test_row_0/C:col10/1733465702853/Put/seqid=0 2024-12-06T06:15:03,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742172_1348 (size=12301) 2024-12-06T06:15:03,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465763734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:03,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465763934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:03,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465763934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:03,938 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:03,938 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:03,943 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/b9c395ed9d674d628a79f4ab37444852 2024-12-06T06:15:03,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465763944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:03,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465763945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:03,949 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:03,949 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/fe885255287d49258c94a4982b053487 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/fe885255287d49258c94a4982b053487 2024-12-06T06:15:03,961 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/fe885255287d49258c94a4982b053487, entries=150, sequenceid=272, filesize=12.0 K 2024-12-06T06:15:03,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/a948356fc961442b8baaac314c0cd708 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a948356fc961442b8baaac314c0cd708 2024-12-06T06:15:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T06:15:03,967 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a948356fc961442b8baaac314c0cd708, entries=150, sequenceid=272, filesize=12.0 K 2024-12-06T06:15:03,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/b9c395ed9d674d628a79f4ab37444852 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b9c395ed9d674d628a79f4ab37444852 2024-12-06T06:15:03,976 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b9c395ed9d674d628a79f4ab37444852, entries=150, sequenceid=272, filesize=12.0 K 2024-12-06T06:15:03,978 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for aa1db8fd546545da53b24cabfd6bfcf1 in 997ms, sequenceid=272, compaction requested=false 2024-12-06T06:15:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-06T06:15:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-06T06:15:03,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-06T06:15:03,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1520 sec 2024-12-06T06:15:03,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.1590 sec 2024-12-06T06:15:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:04,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T06:15:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:04,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/955e6d205a03459c93654495336e3231 is 50, key is test_row_0/A:col10/1733465703092/Put/seqid=0 2024-12-06T06:15:04,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742173_1349 (size=14741) 2024-12-06T06:15:04,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465764307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:04,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465764423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:04,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465764633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:04,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/955e6d205a03459c93654495336e3231 2024-12-06T06:15:04,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/85743109d1d24c0cb1db343436645a65 is 50, key is test_row_0/B:col10/1733465703092/Put/seqid=0 2024-12-06T06:15:04,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742174_1350 (size=12301) 2024-12-06T06:15:04,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/85743109d1d24c0cb1db343436645a65 2024-12-06T06:15:04,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/07a28dbff0d2495aaaa90ccefb4a9e0f is 50, key is test_row_0/C:col10/1733465703092/Put/seqid=0 2024-12-06T06:15:04,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742175_1351 (size=12301) 2024-12-06T06:15:04,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465764943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-06T06:15:04,974 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:04,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-06T06:15:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-06T06:15:04,979 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:04,981 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:04,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-06T06:15:05,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/07a28dbff0d2495aaaa90ccefb4a9e0f 2024-12-06T06:15:05,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/955e6d205a03459c93654495336e3231 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/955e6d205a03459c93654495336e3231 2024-12-06T06:15:05,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/955e6d205a03459c93654495336e3231, entries=200, sequenceid=294, filesize=14.4 K 2024-12-06T06:15:05,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/85743109d1d24c0cb1db343436645a65 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/85743109d1d24c0cb1db343436645a65 2024-12-06T06:15:05,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/85743109d1d24c0cb1db343436645a65, entries=150, sequenceid=294, filesize=12.0 K 2024-12-06T06:15:05,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/07a28dbff0d2495aaaa90ccefb4a9e0f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/07a28dbff0d2495aaaa90ccefb4a9e0f 2024-12-06T06:15:05,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/07a28dbff0d2495aaaa90ccefb4a9e0f, entries=150, sequenceid=294, filesize=12.0 K 2024-12-06T06:15:05,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for aa1db8fd546545da53b24cabfd6bfcf1 in 880ms, sequenceid=294, compaction requested=true 2024-12-06T06:15:05,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:05,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:05,127 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:05,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:05,127 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:05,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:05,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:05,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:05,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:05,129 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:05,129 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:15:05,129 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:05,129 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6b48adbedb9f4f529ffd69d824054870, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/fe885255287d49258c94a4982b053487, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/955e6d205a03459c93654495336e3231] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=39.0 K 2024-12-06T06:15:05,129 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:05,129 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:15:05,129 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:05,129 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/2eaf680df6dc4e779cb3ed31d0041350, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a948356fc961442b8baaac314c0cd708, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/85743109d1d24c0cb1db343436645a65] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=36.6 K 2024-12-06T06:15:05,130 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b48adbedb9f4f529ffd69d824054870, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733465701774 2024-12-06T06:15:05,130 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eaf680df6dc4e779cb3ed31d0041350, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733465701774 2024-12-06T06:15:05,130 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe885255287d49258c94a4982b053487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733465702844 2024-12-06T06:15:05,131 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a948356fc961442b8baaac314c0cd708, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733465702844 2024-12-06T06:15:05,132 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 955e6d205a03459c93654495336e3231, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733465703042 2024-12-06T06:15:05,132 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 85743109d1d24c0cb1db343436645a65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733465703042 2024-12-06T06:15:05,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-06T06:15:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:05,135 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:15:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:05,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/1b2b29bf8ff24bd2b70506be21cb5c4b is 50, key is test_row_0/A:col10/1733465704298/Put/seqid=0 2024-12-06T06:15:05,164 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:05,165 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/1d12373c9555420681b390a20d11dbaf is 50, key is test_row_0/A:col10/1733465703092/Put/seqid=0 2024-12-06T06:15:05,175 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:05,175 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/dcb5c180e5344836bafb5eeb01b65c65 is 50, key is test_row_0/B:col10/1733465703092/Put/seqid=0 2024-12-06T06:15:05,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742177_1353 (size=13051) 2024-12-06T06:15:05,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742176_1352 (size=12301) 2024-12-06T06:15:05,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742178_1354 (size=13051) 2024-12-06T06:15:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-06T06:15:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:05,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:05,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465765581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-06T06:15:05,617 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/1d12373c9555420681b390a20d11dbaf as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1d12373c9555420681b390a20d11dbaf 2024-12-06T06:15:05,625 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into 1d12373c9555420681b390a20d11dbaf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:05,625 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:05,625 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465705127; duration=0sec 2024-12-06T06:15:05,625 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:05,625 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:05,625 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:05,627 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:05,627 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:15:05,627 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:05,627 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e5fdd4f7c237479994ad1e247471e3dd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b9c395ed9d674d628a79f4ab37444852, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/07a28dbff0d2495aaaa90ccefb4a9e0f] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=36.6 K 2024-12-06T06:15:05,628 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5fdd4f7c237479994ad1e247471e3dd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733465701774 2024-12-06T06:15:05,628 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c395ed9d674d628a79f4ab37444852, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733465702844 2024-12-06T06:15:05,628 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07a28dbff0d2495aaaa90ccefb4a9e0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733465703042 2024-12-06T06:15:05,640 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/1b2b29bf8ff24bd2b70506be21cb5c4b 2024-12-06T06:15:05,643 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:05,643 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/df57cd745a094d58bb043ed250038371 is 50, key is test_row_0/C:col10/1733465703092/Put/seqid=0 2024-12-06T06:15:05,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/a59c04406b5946fda7d1f0ea20411528 is 50, key is test_row_0/B:col10/1733465704298/Put/seqid=0 2024-12-06T06:15:05,659 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/dcb5c180e5344836bafb5eeb01b65c65 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/dcb5c180e5344836bafb5eeb01b65c65 2024-12-06T06:15:05,665 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into dcb5c180e5344836bafb5eeb01b65c65(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:05,666 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:05,666 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465705127; duration=0sec 2024-12-06T06:15:05,666 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:05,666 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:05,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742179_1355 (size=13051) 2024-12-06T06:15:05,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742180_1356 (size=12301) 2024-12-06T06:15:05,694 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/a59c04406b5946fda7d1f0ea20411528 2024-12-06T06:15:05,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465765693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:05,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/5ffdb77fcd9a4bab8108bc6327e6a637 is 50, key is test_row_0/C:col10/1733465704298/Put/seqid=0 2024-12-06T06:15:05,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742181_1357 (size=12301) 2024-12-06T06:15:05,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465765904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465765945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465765945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:05,947 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:05,948 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:05,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465765974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:05,979 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:05,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465765974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:05,980 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:06,090 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/df57cd745a094d58bb043ed250038371 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/df57cd745a094d58bb043ed250038371 2024-12-06T06:15:06,099 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into df57cd745a094d58bb043ed250038371(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:06,099 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:06,099 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465705128; duration=0sec 2024-12-06T06:15:06,099 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:06,099 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-06T06:15:06,147 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/5ffdb77fcd9a4bab8108bc6327e6a637 2024-12-06T06:15:06,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/1b2b29bf8ff24bd2b70506be21cb5c4b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1b2b29bf8ff24bd2b70506be21cb5c4b 2024-12-06T06:15:06,158 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1b2b29bf8ff24bd2b70506be21cb5c4b, entries=150, sequenceid=307, filesize=12.0 K 2024-12-06T06:15:06,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/a59c04406b5946fda7d1f0ea20411528 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a59c04406b5946fda7d1f0ea20411528 2024-12-06T06:15:06,167 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a59c04406b5946fda7d1f0ea20411528, entries=150, sequenceid=307, filesize=12.0 K 2024-12-06T06:15:06,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/5ffdb77fcd9a4bab8108bc6327e6a637 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/5ffdb77fcd9a4bab8108bc6327e6a637 2024-12-06T06:15:06,176 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/5ffdb77fcd9a4bab8108bc6327e6a637, entries=150, sequenceid=307, filesize=12.0 K 2024-12-06T06:15:06,177 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aa1db8fd546545da53b24cabfd6bfcf1 in 1042ms, sequenceid=307, compaction requested=false 2024-12-06T06:15:06,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:06,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:06,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-06T06:15:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-06T06:15:06,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-06T06:15:06,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-12-06T06:15:06,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.2080 sec 2024-12-06T06:15:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:06,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:15:06,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:06,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:06,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:06,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:06,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:06,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:06,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/26c399a013fe482b946ce0a783aab757 is 50, key is test_row_0/A:col10/1733465705488/Put/seqid=0 2024-12-06T06:15:06,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742182_1358 (size=12301) 2024-12-06T06:15:06,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/26c399a013fe482b946ce0a783aab757 2024-12-06T06:15:06,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/765ff72e19c04f47a8d68e93cc1463be is 50, key is test_row_0/B:col10/1733465705488/Put/seqid=0 2024-12-06T06:15:06,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465766265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:06,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742183_1359 (size=12301) 2024-12-06T06:15:06,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465766374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:06,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:06,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:06,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:06,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465766583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:06,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/765ff72e19c04f47a8d68e93cc1463be 2024-12-06T06:15:06,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/f50e20ae67ac41f39ca09e45ecaa8507 is 50, key is test_row_0/C:col10/1733465705488/Put/seqid=0 2024-12-06T06:15:06,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742184_1360 (size=12301) 2024-12-06T06:15:06,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/f50e20ae67ac41f39ca09e45ecaa8507 2024-12-06T06:15:06,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/26c399a013fe482b946ce0a783aab757 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/26c399a013fe482b946ce0a783aab757 2024-12-06T06:15:06,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/26c399a013fe482b946ce0a783aab757, entries=150, sequenceid=330, filesize=12.0 K 2024-12-06T06:15:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/765ff72e19c04f47a8d68e93cc1463be as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/765ff72e19c04f47a8d68e93cc1463be 2024-12-06T06:15:06,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/765ff72e19c04f47a8d68e93cc1463be, entries=150, sequenceid=330, filesize=12.0 K 2024-12-06T06:15:06,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/f50e20ae67ac41f39ca09e45ecaa8507 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/f50e20ae67ac41f39ca09e45ecaa8507 2024-12-06T06:15:06,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/f50e20ae67ac41f39ca09e45ecaa8507, entries=150, sequenceid=330, filesize=12.0 K 2024-12-06T06:15:06,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for aa1db8fd546545da53b24cabfd6bfcf1 in 498ms, sequenceid=330, compaction requested=true 2024-12-06T06:15:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:06,714 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:06,714 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:06,715 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:06,715 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:06,715 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:15:06,715 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:15:06,715 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:06,715 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:06,716 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/dcb5c180e5344836bafb5eeb01b65c65, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a59c04406b5946fda7d1f0ea20411528, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/765ff72e19c04f47a8d68e93cc1463be] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=36.8 K 2024-12-06T06:15:06,716 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1d12373c9555420681b390a20d11dbaf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1b2b29bf8ff24bd2b70506be21cb5c4b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/26c399a013fe482b946ce0a783aab757] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=36.8 K 2024-12-06T06:15:06,716 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb5c180e5344836bafb5eeb01b65c65, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733465703042 2024-12-06T06:15:06,716 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d12373c9555420681b390a20d11dbaf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733465703042 2024-12-06T06:15:06,717 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a59c04406b5946fda7d1f0ea20411528, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733465704286 2024-12-06T06:15:06,717 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2b29bf8ff24bd2b70506be21cb5c4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733465704286 2024-12-06T06:15:06,717 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 765ff72e19c04f47a8d68e93cc1463be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733465705488 2024-12-06T06:15:06,717 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26c399a013fe482b946ce0a783aab757, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733465705488 2024-12-06T06:15:06,731 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:06,732 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/f289f94985cb4c41ae60f0804ffc8b8f is 50, key is test_row_0/A:col10/1733465705488/Put/seqid=0 2024-12-06T06:15:06,736 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:06,736 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/8aa664ce4be44930a6ec70d7fb247231 is 50, key is test_row_0/B:col10/1733465705488/Put/seqid=0 2024-12-06T06:15:06,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742185_1361 (size=13153) 2024-12-06T06:15:06,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742186_1362 (size=13153) 2024-12-06T06:15:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:06,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:15:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:06,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/7095f591977d4e4ea3fc7001c37376e3 is 50, key is test_row_0/A:col10/1733465706895/Put/seqid=0 2024-12-06T06:15:06,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742187_1363 (size=14741) 2024-12-06T06:15:06,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/7095f591977d4e4ea3fc7001c37376e3 2024-12-06T06:15:06,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/41b799d3f2414084aa1e680aff5262ac is 50, key is test_row_0/B:col10/1733465706895/Put/seqid=0 2024-12-06T06:15:06,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742188_1364 (size=12301) 2024-12-06T06:15:06,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/41b799d3f2414084aa1e680aff5262ac 2024-12-06T06:15:06,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/28136cf534224de38af6da186ba62042 is 50, key is test_row_0/C:col10/1733465706895/Put/seqid=0 2024-12-06T06:15:06,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742189_1365 (size=12301) 2024-12-06T06:15:07,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465767002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465767114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:07,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-06T06:15:07,123 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:07,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-06T06:15:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-06T06:15:07,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:07,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:07,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:07,157 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/f289f94985cb4c41ae60f0804ffc8b8f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/f289f94985cb4c41ae60f0804ffc8b8f 2024-12-06T06:15:07,158 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/8aa664ce4be44930a6ec70d7fb247231 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8aa664ce4be44930a6ec70d7fb247231 2024-12-06T06:15:07,169 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into f289f94985cb4c41ae60f0804ffc8b8f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:07,169 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 8aa664ce4be44930a6ec70d7fb247231(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:07,169 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:07,169 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:07,169 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465706714; duration=0sec 2024-12-06T06:15:07,169 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465706714; duration=0sec 2024-12-06T06:15:07,169 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:07,169 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:07,169 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:07,169 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:07,169 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:07,170 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:07,170 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:15:07,170 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:07,170 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/df57cd745a094d58bb043ed250038371, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/5ffdb77fcd9a4bab8108bc6327e6a637, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/f50e20ae67ac41f39ca09e45ecaa8507] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=36.8 K 2024-12-06T06:15:07,171 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting df57cd745a094d58bb043ed250038371, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733465703042 2024-12-06T06:15:07,171 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ffdb77fcd9a4bab8108bc6327e6a637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733465704286 2024-12-06T06:15:07,171 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f50e20ae67ac41f39ca09e45ecaa8507, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733465705488 2024-12-06T06:15:07,179 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:07,179 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/27fddb338c134b5ea79b88e5e9ad766c is 50, key is test_row_0/C:col10/1733465705488/Put/seqid=0 2024-12-06T06:15:07,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742190_1366 (size=13153) 2024-12-06T06:15:07,199 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/27fddb338c134b5ea79b88e5e9ad766c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/27fddb338c134b5ea79b88e5e9ad766c 2024-12-06T06:15:07,204 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into 27fddb338c134b5ea79b88e5e9ad766c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:07,204 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:07,204 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465706714; duration=0sec 2024-12-06T06:15:07,204 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:07,204 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-06T06:15:07,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-06T06:15:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:07,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:07,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465767324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:07,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/28136cf534224de38af6da186ba62042 2024-12-06T06:15:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/7095f591977d4e4ea3fc7001c37376e3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/7095f591977d4e4ea3fc7001c37376e3 2024-12-06T06:15:07,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/7095f591977d4e4ea3fc7001c37376e3, entries=200, sequenceid=343, filesize=14.4 K 2024-12-06T06:15:07,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/41b799d3f2414084aa1e680aff5262ac as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/41b799d3f2414084aa1e680aff5262ac 2024-12-06T06:15:07,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/41b799d3f2414084aa1e680aff5262ac, entries=150, sequenceid=343, filesize=12.0 K 2024-12-06T06:15:07,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/28136cf534224de38af6da186ba62042 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/28136cf534224de38af6da186ba62042 2024-12-06T06:15:07,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/28136cf534224de38af6da186ba62042, entries=150, sequenceid=343, filesize=12.0 K 2024-12-06T06:15:07,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aa1db8fd546545da53b24cabfd6bfcf1 in 459ms, sequenceid=343, compaction requested=false 2024-12-06T06:15:07,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:07,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-06T06:15:07,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:07,434 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T06:15:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/6e0d4dc7245a4451b0015915354bd2fb is 50, key is test_row_0/A:col10/1733465707000/Put/seqid=0 2024-12-06T06:15:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-06T06:15:07,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742191_1367 (size=12301) 2024-12-06T06:15:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:07,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:07,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465767689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-06T06:15:07,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465767804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:07,849 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/6e0d4dc7245a4451b0015915354bd2fb 2024-12-06T06:15:07,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/3876969e69534eb390cd6015b1686adc is 50, key is test_row_0/B:col10/1733465707000/Put/seqid=0 2024-12-06T06:15:07,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742192_1368 (size=12301) 2024-12-06T06:15:08,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:08,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465768013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-06T06:15:08,278 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/3876969e69534eb390cd6015b1686adc 2024-12-06T06:15:08,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/38030f77750f420b96699466d0860574 is 50, key is test_row_0/C:col10/1733465707000/Put/seqid=0 2024-12-06T06:15:08,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742193_1369 (size=12301) 2024-12-06T06:15:08,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465768324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:08,691 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/38030f77750f420b96699466d0860574 2024-12-06T06:15:08,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/6e0d4dc7245a4451b0015915354bd2fb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6e0d4dc7245a4451b0015915354bd2fb 2024-12-06T06:15:08,699 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6e0d4dc7245a4451b0015915354bd2fb, entries=150, sequenceid=365, filesize=12.0 K 2024-12-06T06:15:08,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/3876969e69534eb390cd6015b1686adc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/3876969e69534eb390cd6015b1686adc 2024-12-06T06:15:08,704 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/3876969e69534eb390cd6015b1686adc, entries=150, sequenceid=365, filesize=12.0 K 2024-12-06T06:15:08,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/38030f77750f420b96699466d0860574 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/38030f77750f420b96699466d0860574 2024-12-06T06:15:08,713 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/38030f77750f420b96699466d0860574, entries=150, sequenceid=365, filesize=12.0 K 2024-12-06T06:15:08,713 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for aa1db8fd546545da53b24cabfd6bfcf1 in 1279ms, sequenceid=365, compaction requested=true 2024-12-06T06:15:08,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:08,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:08,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-06T06:15:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-06T06:15:08,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-06T06:15:08,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5880 sec 2024-12-06T06:15:08,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.5940 sec 2024-12-06T06:15:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:08,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-06T06:15:08,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:08,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:08,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:08,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:08,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:08,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:08,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/9fa099575ba441c49e2eeeee14c02455 is 50, key is test_row_0/A:col10/1733465708835/Put/seqid=0 2024-12-06T06:15:08,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742194_1370 (size=14741) 2024-12-06T06:15:08,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/9fa099575ba441c49e2eeeee14c02455 2024-12-06T06:15:08,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/6a8d9a0cda2e4c7893db50f1edd623f3 is 50, key is test_row_0/B:col10/1733465708835/Put/seqid=0 2024-12-06T06:15:08,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742195_1371 (size=12301) 2024-12-06T06:15:08,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/6a8d9a0cda2e4c7893db50f1edd623f3 2024-12-06T06:15:08,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/eba417caca3e4787838a74bda123db81 is 50, key is test_row_0/C:col10/1733465708835/Put/seqid=0 2024-12-06T06:15:08,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742196_1372 (size=12301) 2024-12-06T06:15:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465768925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:09,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465769034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:09,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465769243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-06T06:15:09,274 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:09,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-06T06:15:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-06T06:15:09,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:09,278 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:09,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:09,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/eba417caca3e4787838a74bda123db81 2024-12-06T06:15:09,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/9fa099575ba441c49e2eeeee14c02455 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/9fa099575ba441c49e2eeeee14c02455 2024-12-06T06:15:09,290 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T06:15:09,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/9fa099575ba441c49e2eeeee14c02455, entries=200, sequenceid=379, filesize=14.4 K 2024-12-06T06:15:09,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/6a8d9a0cda2e4c7893db50f1edd623f3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6a8d9a0cda2e4c7893db50f1edd623f3 2024-12-06T06:15:09,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6a8d9a0cda2e4c7893db50f1edd623f3, entries=150, sequenceid=379, filesize=12.0 K 2024-12-06T06:15:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/eba417caca3e4787838a74bda123db81 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/eba417caca3e4787838a74bda123db81 2024-12-06T06:15:09,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/eba417caca3e4787838a74bda123db81, entries=150, sequenceid=379, filesize=12.0 K 2024-12-06T06:15:09,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for aa1db8fd546545da53b24cabfd6bfcf1 in 465ms, sequenceid=379, compaction requested=true 2024-12-06T06:15:09,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:09,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:09,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:09,302 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:09,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:09,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:09,302 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:09,303 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:09,303 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:09,303 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:15:09,303 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:15:09,304 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:09,304 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:09,304 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8aa664ce4be44930a6ec70d7fb247231, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/41b799d3f2414084aa1e680aff5262ac, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/3876969e69534eb390cd6015b1686adc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6a8d9a0cda2e4c7893db50f1edd623f3] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=48.9 K 2024-12-06T06:15:09,304 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/f289f94985cb4c41ae60f0804ffc8b8f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/7095f591977d4e4ea3fc7001c37376e3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6e0d4dc7245a4451b0015915354bd2fb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/9fa099575ba441c49e2eeeee14c02455] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=53.6 K 2024-12-06T06:15:09,305 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa664ce4be44930a6ec70d7fb247231, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733465705488 2024-12-06T06:15:09,305 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f289f94985cb4c41ae60f0804ffc8b8f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733465705488 2024-12-06T06:15:09,305 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b799d3f2414084aa1e680aff5262ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733465706255 2024-12-06T06:15:09,305 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7095f591977d4e4ea3fc7001c37376e3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733465706232 2024-12-06T06:15:09,305 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3876969e69534eb390cd6015b1686adc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733465706988 2024-12-06T06:15:09,305 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e0d4dc7245a4451b0015915354bd2fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733465706988 2024-12-06T06:15:09,306 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8d9a0cda2e4c7893db50f1edd623f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733465707681 2024-12-06T06:15:09,306 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fa099575ba441c49e2eeeee14c02455, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733465707654 2024-12-06T06:15:09,317 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:09,318 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/22352606b6d049078f36ef26df9250fc is 50, key is test_row_0/A:col10/1733465708835/Put/seqid=0 2024-12-06T06:15:09,321 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:09,321 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/270c830392c946a5ac4bd2b45c6207c9 is 50, key is test_row_0/B:col10/1733465708835/Put/seqid=0 2024-12-06T06:15:09,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742197_1373 (size=13289) 2024-12-06T06:15:09,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742198_1374 (size=13289) 2024-12-06T06:15:09,335 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/270c830392c946a5ac4bd2b45c6207c9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/270c830392c946a5ac4bd2b45c6207c9 2024-12-06T06:15:09,341 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 270c830392c946a5ac4bd2b45c6207c9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:09,341 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:09,341 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=12, startTime=1733465709302; duration=0sec 2024-12-06T06:15:09,341 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:09,341 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:09,341 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:09,343 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:09,343 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:15:09,343 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:09,343 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/27fddb338c134b5ea79b88e5e9ad766c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/28136cf534224de38af6da186ba62042, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/38030f77750f420b96699466d0860574, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/eba417caca3e4787838a74bda123db81] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=48.9 K 2024-12-06T06:15:09,343 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fddb338c134b5ea79b88e5e9ad766c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733465705488 2024-12-06T06:15:09,344 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 28136cf534224de38af6da186ba62042, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733465706255 2024-12-06T06:15:09,344 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 38030f77750f420b96699466d0860574, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733465706988 2024-12-06T06:15:09,344 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting eba417caca3e4787838a74bda123db81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733465707681 2024-12-06T06:15:09,356 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:09,357 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/a42b00e716f84fbda1e1f3d61c7b52cd is 50, key is test_row_0/C:col10/1733465708835/Put/seqid=0 2024-12-06T06:15:09,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742199_1375 (size=13289) 2024-12-06T06:15:09,372 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/a42b00e716f84fbda1e1f3d61c7b52cd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a42b00e716f84fbda1e1f3d61c7b52cd 2024-12-06T06:15:09,377 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into a42b00e716f84fbda1e1f3d61c7b52cd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:09,377 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:09,377 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=12, startTime=1733465709302; duration=0sec 2024-12-06T06:15:09,377 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:09,377 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-06T06:15:09,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-06T06:15:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:09,432 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:15:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:09,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/d80baf1e82fd47328c4abeb579a0b97b is 50, key is test_row_0/A:col10/1733465708915/Put/seqid=0 2024-12-06T06:15:09,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742200_1376 (size=12301) 2024-12-06T06:15:09,464 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/d80baf1e82fd47328c4abeb579a0b97b 2024-12-06T06:15:09,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/6d6a82131e884087b5b20be066c2896a is 50, key is test_row_0/B:col10/1733465708915/Put/seqid=0 2024-12-06T06:15:09,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742201_1377 (size=12301) 2024-12-06T06:15:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:09,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-06T06:15:09,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465769598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:09,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465769714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:09,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:09,738 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/22352606b6d049078f36ef26df9250fc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/22352606b6d049078f36ef26df9250fc 2024-12-06T06:15:09,745 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into 22352606b6d049078f36ef26df9250fc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:09,745 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:09,745 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=12, startTime=1733465709302; duration=0sec 2024-12-06T06:15:09,745 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:09,745 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:09,886 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/6d6a82131e884087b5b20be066c2896a 2024-12-06T06:15:09,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/a61d9520d5054824a9503cda0b434645 is 50, key is test_row_0/C:col10/1733465708915/Put/seqid=0 2024-12-06T06:15:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-06T06:15:09,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742202_1378 (size=12301) 2024-12-06T06:15:09,917 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/a61d9520d5054824a9503cda0b434645 2024-12-06T06:15:09,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465769924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:09,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/d80baf1e82fd47328c4abeb579a0b97b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d80baf1e82fd47328c4abeb579a0b97b 2024-12-06T06:15:09,942 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d80baf1e82fd47328c4abeb579a0b97b, entries=150, sequenceid=399, filesize=12.0 K 2024-12-06T06:15:09,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/6d6a82131e884087b5b20be066c2896a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6d6a82131e884087b5b20be066c2896a 2024-12-06T06:15:09,956 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6d6a82131e884087b5b20be066c2896a, entries=150, sequenceid=399, filesize=12.0 K 2024-12-06T06:15:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/a61d9520d5054824a9503cda0b434645 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a61d9520d5054824a9503cda0b434645 2024-12-06T06:15:09,964 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a61d9520d5054824a9503cda0b434645, entries=150, sequenceid=399, filesize=12.0 K 2024-12-06T06:15:09,965 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for aa1db8fd546545da53b24cabfd6bfcf1 in 533ms, sequenceid=399, compaction requested=false 2024-12-06T06:15:09,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:09,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:09,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-06T06:15:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-06T06:15:09,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:15:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:09,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-06T06:15:09,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 689 msec 2024-12-06T06:15:09,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 696 msec 2024-12-06T06:15:09,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/c49f2dafec574179985346b09ecf0815 is 50, key is test_row_0/A:col10/1733465709592/Put/seqid=0 2024-12-06T06:15:10,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742203_1379 (size=14741) 2024-12-06T06:15:10,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465770016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465770019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465770023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58244 deadline: 1733465770024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,029 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in aa1db8fd546545da53b24cabfd6bfcf1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465770134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465770134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465770134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465770233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465770343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465770344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465770344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-06T06:15:10,413 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:10,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-12-06T06:15:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-06T06:15:10,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:10,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:10,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:10,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/c49f2dafec574179985346b09ecf0815 2024-12-06T06:15:10,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/f7ac5e3bd52342a69b9bd6b34fd26101 is 50, key is test_row_0/B:col10/1733465709592/Put/seqid=0 2024-12-06T06:15:10,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742204_1380 (size=12301) 2024-12-06T06:15:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-06T06:15:10,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-06T06:15:10,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:10,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:10,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:10,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:10,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465770653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465770653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465770653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-06T06:15:10,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:10,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:10,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:10,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:10,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-06T06:15:10,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58274 deadline: 1733465770743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:10,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/f7ac5e3bd52342a69b9bd6b34fd26101 2024-12-06T06:15:10,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/ed10c57ad782489aadf407a60608b0ae is 50, key is test_row_0/C:col10/1733465709592/Put/seqid=0 2024-12-06T06:15:10,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742205_1381 (size=12301) 2024-12-06T06:15:10,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/ed10c57ad782489aadf407a60608b0ae 2024-12-06T06:15:10,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/c49f2dafec574179985346b09ecf0815 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/c49f2dafec574179985346b09ecf0815 2024-12-06T06:15:10,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/c49f2dafec574179985346b09ecf0815, entries=200, sequenceid=415, filesize=14.4 K 2024-12-06T06:15:10,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/f7ac5e3bd52342a69b9bd6b34fd26101 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/f7ac5e3bd52342a69b9bd6b34fd26101 2024-12-06T06:15:10,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/f7ac5e3bd52342a69b9bd6b34fd26101, entries=150, sequenceid=415, filesize=12.0 K 2024-12-06T06:15:10,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/ed10c57ad782489aadf407a60608b0ae as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ed10c57ad782489aadf407a60608b0ae 2024-12-06T06:15:10,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ed10c57ad782489aadf407a60608b0ae, entries=150, sequenceid=415, filesize=12.0 K 2024-12-06T06:15:10,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for aa1db8fd546545da53b24cabfd6bfcf1 in 899ms, sequenceid=415, compaction requested=true 2024-12-06T06:15:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:10,868 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:10,868 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:10,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:10,869 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:10,870 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:15:10,870 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:10,870 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:10,870 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:15:10,870 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/270c830392c946a5ac4bd2b45c6207c9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6d6a82131e884087b5b20be066c2896a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/f7ac5e3bd52342a69b9bd6b34fd26101] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=37.0 K 2024-12-06T06:15:10,870 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:10,870 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/22352606b6d049078f36ef26df9250fc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d80baf1e82fd47328c4abeb579a0b97b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/c49f2dafec574179985346b09ecf0815] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=39.4 K 2024-12-06T06:15:10,870 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 270c830392c946a5ac4bd2b45c6207c9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733465707681 2024-12-06T06:15:10,870 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22352606b6d049078f36ef26df9250fc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733465707681 2024-12-06T06:15:10,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d6a82131e884087b5b20be066c2896a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465708907 2024-12-06T06:15:10,871 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d80baf1e82fd47328c4abeb579a0b97b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465708907 2024-12-06T06:15:10,871 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49f2dafec574179985346b09ecf0815, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733465709583 2024-12-06T06:15:10,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ac5e3bd52342a69b9bd6b34fd26101, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733465709592 2024-12-06T06:15:10,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-06T06:15:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:10,876 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T06:15:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:10,878 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:10,879 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/2f97a525a69a4a5a9e2ee71a1a17da33 is 50, key is test_row_0/A:col10/1733465709592/Put/seqid=0 2024-12-06T06:15:10,879 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:10,880 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/49a42e8e23d6408a8edc7de5f685d9ed is 50, key is test_row_0/B:col10/1733465709592/Put/seqid=0 2024-12-06T06:15:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/582024b1619e430d9eb073775d433671 is 50, key is test_row_0/A:col10/1733465709992/Put/seqid=0 2024-12-06T06:15:10,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742206_1382 (size=13391) 2024-12-06T06:15:10,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742207_1383 (size=13391) 2024-12-06T06:15:10,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742208_1384 (size=12301) 2024-12-06T06:15:10,908 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/582024b1619e430d9eb073775d433671 2024-12-06T06:15:10,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/0aa97632686c41109a48074eb5059f4a is 50, key is test_row_0/B:col10/1733465709992/Put/seqid=0 2024-12-06T06:15:10,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742209_1385 (size=12301) 2024-12-06T06:15:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-06T06:15:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:11,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. as already flushing 2024-12-06T06:15:11,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465771182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:11,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:11,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465771187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465771187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:11,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58202 deadline: 1733465771295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:11,301 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/2f97a525a69a4a5a9e2ee71a1a17da33 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/2f97a525a69a4a5a9e2ee71a1a17da33 2024-12-06T06:15:11,301 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/49a42e8e23d6408a8edc7de5f685d9ed as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49a42e8e23d6408a8edc7de5f685d9ed 2024-12-06T06:15:11,305 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 49a42e8e23d6408a8edc7de5f685d9ed(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:11,305 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into 2f97a525a69a4a5a9e2ee71a1a17da33(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:11,305 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:11,305 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:11,305 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465710868; duration=0sec 2024-12-06T06:15:11,305 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465710868; duration=0sec 2024-12-06T06:15:11,306 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:11,306 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:11,306 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:11,306 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:11,306 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:11,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58188 deadline: 1733465771303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733465771304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:11,310 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:11,310 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:15:11,310 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:11,310 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a42b00e716f84fbda1e1f3d61c7b52cd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a61d9520d5054824a9503cda0b434645, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ed10c57ad782489aadf407a60608b0ae] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=37.0 K 2024-12-06T06:15:11,310 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a42b00e716f84fbda1e1f3d61c7b52cd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733465707681 2024-12-06T06:15:11,310 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a61d9520d5054824a9503cda0b434645, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733465708907 2024-12-06T06:15:11,311 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed10c57ad782489aadf407a60608b0ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733465709592 2024-12-06T06:15:11,317 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:11,318 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/e885e7e14d1b431e91bbe0d3910ef91a is 50, key is test_row_0/C:col10/1733465709592/Put/seqid=0 2024-12-06T06:15:11,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742210_1386 (size=13391) 2024-12-06T06:15:11,322 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/0aa97632686c41109a48074eb5059f4a 2024-12-06T06:15:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/d245d41d42664bec9711bcacf43452b2 is 50, key is test_row_0/C:col10/1733465709992/Put/seqid=0 2024-12-06T06:15:11,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742211_1387 (size=12301) 2024-12-06T06:15:11,341 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/d245d41d42664bec9711bcacf43452b2 2024-12-06T06:15:11,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/582024b1619e430d9eb073775d433671 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/582024b1619e430d9eb073775d433671 2024-12-06T06:15:11,348 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/582024b1619e430d9eb073775d433671, entries=150, sequenceid=432, filesize=12.0 K 2024-12-06T06:15:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/0aa97632686c41109a48074eb5059f4a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/0aa97632686c41109a48074eb5059f4a 2024-12-06T06:15:11,352 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/0aa97632686c41109a48074eb5059f4a, entries=150, sequenceid=432, filesize=12.0 K 2024-12-06T06:15:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/d245d41d42664bec9711bcacf43452b2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d245d41d42664bec9711bcacf43452b2 2024-12-06T06:15:11,356 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d245d41d42664bec9711bcacf43452b2, entries=150, sequenceid=432, filesize=12.0 K 2024-12-06T06:15:11,357 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for aa1db8fd546545da53b24cabfd6bfcf1 in 482ms, sequenceid=432, compaction requested=false 2024-12-06T06:15:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-12-06T06:15:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-12-06T06:15:11,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-06T06:15:11,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 943 msec 2024-12-06T06:15:11,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 947 msec 2024-12-06T06:15:11,450 INFO [Thread-1313 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1313. 2024-12-06T06:15:11,450 DEBUG [Thread-1313 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,450 DEBUG [Thread-1313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,450 DEBUG [Thread-1313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,451 INFO [Thread-1317 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1317. 2024-12-06T06:15:11,451 DEBUG [Thread-1317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,451 DEBUG [Thread-1317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,451 DEBUG [Thread-1317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,452 INFO [Thread-1315 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1315. 2024-12-06T06:15:11,452 DEBUG [Thread-1315 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,452 DEBUG [Thread-1315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,452 DEBUG [Thread-1315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,453 INFO [Thread-1311 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1311. 2024-12-06T06:15:11,453 DEBUG [Thread-1311 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,453 DEBUG [Thread-1311 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,453 DEBUG [Thread-1311 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,454 INFO [Thread-1309 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1309. 2024-12-06T06:15:11,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,454 DEBUG [Thread-1309 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,454 DEBUG [Thread-1309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,454 DEBUG [Thread-1309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:11,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-06T06:15:11,506 INFO [Thread-1306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1306. 2024-12-06T06:15:11,506 DEBUG [Thread-1306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,506 DEBUG [Thread-1306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:11,506 DEBUG [Thread-1306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:11,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:11,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:11,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:11,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:11,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/49005dad29384614ad15fd7dad8977d1 is 50, key is test_row_0/A:col10/1733465711179/Put/seqid=0 2024-12-06T06:15:11,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742212_1388 (size=12301) 2024-12-06T06:15:11,515 INFO [Thread-1302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1302. 2024-12-06T06:15:11,515 INFO [Thread-1298 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1298. 2024-12-06T06:15:11,515 DEBUG [Thread-1302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,515 DEBUG [Thread-1298 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,515 DEBUG [Thread-1302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,515 DEBUG [Thread-1298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,515 DEBUG [Thread-1302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,515 DEBUG [Thread-1298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-06T06:15:11,554 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:11,726 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/e885e7e14d1b431e91bbe0d3910ef91a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e885e7e14d1b431e91bbe0d3910ef91a 2024-12-06T06:15:11,729 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into e885e7e14d1b431e91bbe0d3910ef91a(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:11,729 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:11,729 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465710868; duration=0sec 2024-12-06T06:15:11,729 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:11,729 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:11,755 INFO [Thread-1304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1304. 2024-12-06T06:15:11,755 DEBUG [Thread-1304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:11,755 DEBUG [Thread-1304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,755 DEBUG [Thread-1304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:11,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:11,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/49005dad29384614ad15fd7dad8977d1 2024-12-06T06:15:11,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e5ebd39fec1342c4a27b4e6e81715cdf is 50, key is test_row_0/B:col10/1733465711179/Put/seqid=0 2024-12-06T06:15:11,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742213_1389 (size=12301) 2024-12-06T06:15:12,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e5ebd39fec1342c4a27b4e6e81715cdf 2024-12-06T06:15:12,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/b6c73e1619c7490ab2d60f6672d7ed36 is 50, key is test_row_0/C:col10/1733465711179/Put/seqid=0 2024-12-06T06:15:12,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742214_1390 (size=12301) 2024-12-06T06:15:12,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/b6c73e1619c7490ab2d60f6672d7ed36 2024-12-06T06:15:12,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/49005dad29384614ad15fd7dad8977d1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/49005dad29384614ad15fd7dad8977d1 2024-12-06T06:15:12,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/49005dad29384614ad15fd7dad8977d1, entries=150, sequenceid=450, filesize=12.0 K 2024-12-06T06:15:12,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e5ebd39fec1342c4a27b4e6e81715cdf as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e5ebd39fec1342c4a27b4e6e81715cdf 2024-12-06T06:15:12,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e5ebd39fec1342c4a27b4e6e81715cdf, entries=150, sequenceid=450, filesize=12.0 K 2024-12-06T06:15:12,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/b6c73e1619c7490ab2d60f6672d7ed36 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b6c73e1619c7490ab2d60f6672d7ed36 2024-12-06T06:15:12,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b6c73e1619c7490ab2d60f6672d7ed36, entries=150, sequenceid=450, filesize=12.0 K 2024-12-06T06:15:12,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for aa1db8fd546545da53b24cabfd6bfcf1 in 1242ms, sequenceid=450, compaction requested=true 2024-12-06T06:15:12,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:12,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:12,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:12,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:12,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:12,747 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:12,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa1db8fd546545da53b24cabfd6bfcf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:12,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:12,747 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:12,748 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:12,748 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:12,748 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/B is initiating minor compaction (all files) 2024-12-06T06:15:12,748 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/A is initiating minor compaction (all files) 2024-12-06T06:15:12,748 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/B in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:12,748 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/A in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:12,748 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49a42e8e23d6408a8edc7de5f685d9ed, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/0aa97632686c41109a48074eb5059f4a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e5ebd39fec1342c4a27b4e6e81715cdf] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=37.1 K 2024-12-06T06:15:12,748 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/2f97a525a69a4a5a9e2ee71a1a17da33, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/582024b1619e430d9eb073775d433671, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/49005dad29384614ad15fd7dad8977d1] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=37.1 K 2024-12-06T06:15:12,749 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a42e8e23d6408a8edc7de5f685d9ed, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733465709592 2024-12-06T06:15:12,749 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f97a525a69a4a5a9e2ee71a1a17da33, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733465709592 2024-12-06T06:15:12,749 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa97632686c41109a48074eb5059f4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733465709992 2024-12-06T06:15:12,749 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 582024b1619e430d9eb073775d433671, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733465709992 2024-12-06T06:15:12,749 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ebd39fec1342c4a27b4e6e81715cdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733465711179 2024-12-06T06:15:12,749 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49005dad29384614ad15fd7dad8977d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733465711179 2024-12-06T06:15:12,755 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#A#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:12,755 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:12,755 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/baf5a63b28974ad187ee2f76493d9521 is 50, key is test_row_0/A:col10/1733465711179/Put/seqid=0 2024-12-06T06:15:12,755 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/99b4f1e74e1d472cb4ecc223d6d82308 is 50, key is test_row_0/B:col10/1733465711179/Put/seqid=0 2024-12-06T06:15:12,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742215_1391 (size=13493) 2024-12-06T06:15:12,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742216_1392 (size=13493) 2024-12-06T06:15:13,163 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/baf5a63b28974ad187ee2f76493d9521 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/baf5a63b28974ad187ee2f76493d9521 2024-12-06T06:15:13,163 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/99b4f1e74e1d472cb4ecc223d6d82308 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/99b4f1e74e1d472cb4ecc223d6d82308 2024-12-06T06:15:13,166 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/A of aa1db8fd546545da53b24cabfd6bfcf1 into baf5a63b28974ad187ee2f76493d9521(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:13,166 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/B of aa1db8fd546545da53b24cabfd6bfcf1 into 99b4f1e74e1d472cb4ecc223d6d82308(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:13,166 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:13,166 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:13,166 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/A, priority=13, startTime=1733465712747; duration=0sec 2024-12-06T06:15:13,166 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/B, priority=13, startTime=1733465712747; duration=0sec 2024-12-06T06:15:13,166 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:13,166 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:13,166 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:A 2024-12-06T06:15:13,166 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:B 2024-12-06T06:15:13,166 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:13,167 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:13,167 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): aa1db8fd546545da53b24cabfd6bfcf1/C is initiating minor compaction (all files) 2024-12-06T06:15:13,167 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa1db8fd546545da53b24cabfd6bfcf1/C in TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:13,167 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e885e7e14d1b431e91bbe0d3910ef91a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d245d41d42664bec9711bcacf43452b2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b6c73e1619c7490ab2d60f6672d7ed36] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp, totalSize=37.1 K 2024-12-06T06:15:13,167 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e885e7e14d1b431e91bbe0d3910ef91a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733465709592 2024-12-06T06:15:13,168 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d245d41d42664bec9711bcacf43452b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733465709992 2024-12-06T06:15:13,168 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6c73e1619c7490ab2d60f6672d7ed36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733465711179 2024-12-06T06:15:13,173 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa1db8fd546545da53b24cabfd6bfcf1#C#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:13,174 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/4c25e8fe0a104b4eb9b9941f8afff737 is 50, key is test_row_0/C:col10/1733465711179/Put/seqid=0 2024-12-06T06:15:13,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742217_1393 (size=13493) 2024-12-06T06:15:13,581 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/4c25e8fe0a104b4eb9b9941f8afff737 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/4c25e8fe0a104b4eb9b9941f8afff737 2024-12-06T06:15:13,584 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa1db8fd546545da53b24cabfd6bfcf1/C of aa1db8fd546545da53b24cabfd6bfcf1 into 4c25e8fe0a104b4eb9b9941f8afff737(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:13,584 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa1db8fd546545da53b24cabfd6bfcf1: 2024-12-06T06:15:13,584 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., storeName=aa1db8fd546545da53b24cabfd6bfcf1/C, priority=13, startTime=1733465712747; duration=0sec 2024-12-06T06:15:13,584 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:13,584 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa1db8fd546545da53b24cabfd6bfcf1:C 2024-12-06T06:15:20,065 INFO [Thread-1300 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1300. 2024-12-06T06:15:20,065 DEBUG [Thread-1300 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:20,065 DEBUG [Thread-1300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:20,065 DEBUG [Thread-1300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:20,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3094 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2862 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2734 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3070 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2824 2024-12-06T06:15:20,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T06:15:20,065 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T06:15:20,066 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:15:20,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:20,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:20,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:20,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T06:15:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-06T06:15:20,069 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465720069"}]},"ts":"1733465720069"} 2024-12-06T06:15:20,070 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T06:15:20,070 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T06:15:20,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:15:20,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, UNASSIGN}] 2024-12-06T06:15:20,073 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, UNASSIGN 2024-12-06T06:15:20,073 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=aa1db8fd546545da53b24cabfd6bfcf1, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:20,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, UNASSIGN because future has completed 2024-12-06T06:15:20,075 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:15:20,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:15:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-06T06:15:20,227 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:20,227 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:15:20,227 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing aa1db8fd546545da53b24cabfd6bfcf1, disabling compactions & flushes 2024-12-06T06:15:20,227 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:20,227 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:20,227 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. after waiting 0 ms 2024-12-06T06:15:20,227 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:20,227 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing aa1db8fd546545da53b24cabfd6bfcf1 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-06T06:15:20,228 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=A 2024-12-06T06:15:20,228 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:20,228 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=B 2024-12-06T06:15:20,228 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:20,228 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa1db8fd546545da53b24cabfd6bfcf1, store=C 2024-12-06T06:15:20,228 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:20,231 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/b526368ce79140b9b39d5a23553fa635 is 50, key is test_row_0/A:col10/1733465711513/Put/seqid=0 2024-12-06T06:15:20,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742218_1394 (size=12301) 2024-12-06T06:15:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-06T06:15:20,635 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/b526368ce79140b9b39d5a23553fa635 2024-12-06T06:15:20,642 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e85e7e0ac1fd46709d1c45022f8d422b is 50, key is test_row_0/B:col10/1733465711513/Put/seqid=0 2024-12-06T06:15:20,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742219_1395 (size=12301) 2024-12-06T06:15:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-06T06:15:21,047 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e85e7e0ac1fd46709d1c45022f8d422b 2024-12-06T06:15:21,054 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/450bbbd1969349db94aa71c7519d6703 is 50, key is test_row_0/C:col10/1733465711513/Put/seqid=0 2024-12-06T06:15:21,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742220_1396 (size=12301) 2024-12-06T06:15:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-06T06:15:21,457 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/450bbbd1969349db94aa71c7519d6703 2024-12-06T06:15:21,461 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/A/b526368ce79140b9b39d5a23553fa635 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b526368ce79140b9b39d5a23553fa635 2024-12-06T06:15:21,464 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b526368ce79140b9b39d5a23553fa635, entries=150, sequenceid=461, filesize=12.0 K 2024-12-06T06:15:21,464 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/B/e85e7e0ac1fd46709d1c45022f8d422b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e85e7e0ac1fd46709d1c45022f8d422b 2024-12-06T06:15:21,467 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e85e7e0ac1fd46709d1c45022f8d422b, entries=150, sequenceid=461, filesize=12.0 K 2024-12-06T06:15:21,467 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/.tmp/C/450bbbd1969349db94aa71c7519d6703 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/450bbbd1969349db94aa71c7519d6703 2024-12-06T06:15:21,470 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/450bbbd1969349db94aa71c7519d6703, entries=150, sequenceid=461, filesize=12.0 K 2024-12-06T06:15:21,471 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for aa1db8fd546545da53b24cabfd6bfcf1 in 1244ms, sequenceid=461, compaction requested=false 2024-12-06T06:15:21,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/27a3339547044648baad2fe550682b34, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7140dbd7fd422c8f949bac4788f175, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/ffd555fbd5ff47c08f722fde74de28cc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/e10039a6daee454e95895f28b1d34e67, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b51940ac1b324b2baf78de7f63c51ba8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/18a687b4e98d4d3abfd4ce3ac757707a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/513418c8568145a7950e388b4f099462, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d1b29741b58841a3a69757171c4abca2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/29d76126789c47c881ae42f1d42a7801, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b4be0219c2b44cf3a3a97d206cc92123, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/952e480a842f481592f337dcd94a496c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3b3a03e3ad7241a1bb98d821d8d9c6c0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3bbc8f0bdf05416cb9afba8a76f4fd73, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3399ba691cb94349b3361e608608565a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/5b46c3e891e14c88b7ad8a30c217db34, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0c7d5846943d404d960f67230f8a130d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/8cc451244fa64963bac12c14b831ce1f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0fdc07222ac1485eb8fbd6b5910df231, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6b48adbedb9f4f529ffd69d824054870, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7349db8e344e2ebfe9f9c305f5f615, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/fe885255287d49258c94a4982b053487, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/955e6d205a03459c93654495336e3231, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1d12373c9555420681b390a20d11dbaf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1b2b29bf8ff24bd2b70506be21cb5c4b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/f289f94985cb4c41ae60f0804ffc8b8f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/26c399a013fe482b946ce0a783aab757, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/7095f591977d4e4ea3fc7001c37376e3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6e0d4dc7245a4451b0015915354bd2fb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/9fa099575ba441c49e2eeeee14c02455, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/22352606b6d049078f36ef26df9250fc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d80baf1e82fd47328c4abeb579a0b97b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/c49f2dafec574179985346b09ecf0815, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/2f97a525a69a4a5a9e2ee71a1a17da33, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/582024b1619e430d9eb073775d433671, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/49005dad29384614ad15fd7dad8977d1] to archive 2024-12-06T06:15:21,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:21,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/27a3339547044648baad2fe550682b34 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/27a3339547044648baad2fe550682b34 2024-12-06T06:15:21,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7140dbd7fd422c8f949bac4788f175 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7140dbd7fd422c8f949bac4788f175 2024-12-06T06:15:21,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/ffd555fbd5ff47c08f722fde74de28cc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/ffd555fbd5ff47c08f722fde74de28cc 2024-12-06T06:15:21,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/e10039a6daee454e95895f28b1d34e67 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/e10039a6daee454e95895f28b1d34e67 2024-12-06T06:15:21,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b51940ac1b324b2baf78de7f63c51ba8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b51940ac1b324b2baf78de7f63c51ba8 2024-12-06T06:15:21,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/18a687b4e98d4d3abfd4ce3ac757707a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/18a687b4e98d4d3abfd4ce3ac757707a 2024-12-06T06:15:21,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/513418c8568145a7950e388b4f099462 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/513418c8568145a7950e388b4f099462 2024-12-06T06:15:21,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d1b29741b58841a3a69757171c4abca2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d1b29741b58841a3a69757171c4abca2 2024-12-06T06:15:21,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/29d76126789c47c881ae42f1d42a7801 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/29d76126789c47c881ae42f1d42a7801 2024-12-06T06:15:21,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b4be0219c2b44cf3a3a97d206cc92123 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b4be0219c2b44cf3a3a97d206cc92123 2024-12-06T06:15:21,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/952e480a842f481592f337dcd94a496c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/952e480a842f481592f337dcd94a496c 2024-12-06T06:15:21,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3b3a03e3ad7241a1bb98d821d8d9c6c0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3b3a03e3ad7241a1bb98d821d8d9c6c0 2024-12-06T06:15:21,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3bbc8f0bdf05416cb9afba8a76f4fd73 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3bbc8f0bdf05416cb9afba8a76f4fd73 2024-12-06T06:15:21,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3399ba691cb94349b3361e608608565a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/3399ba691cb94349b3361e608608565a 2024-12-06T06:15:21,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/5b46c3e891e14c88b7ad8a30c217db34 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/5b46c3e891e14c88b7ad8a30c217db34 2024-12-06T06:15:21,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0c7d5846943d404d960f67230f8a130d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0c7d5846943d404d960f67230f8a130d 2024-12-06T06:15:21,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/8cc451244fa64963bac12c14b831ce1f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/8cc451244fa64963bac12c14b831ce1f 2024-12-06T06:15:21,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0fdc07222ac1485eb8fbd6b5910df231 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/0fdc07222ac1485eb8fbd6b5910df231 2024-12-06T06:15:21,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6b48adbedb9f4f529ffd69d824054870 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6b48adbedb9f4f529ffd69d824054870 2024-12-06T06:15:21,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7349db8e344e2ebfe9f9c305f5f615 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/df7349db8e344e2ebfe9f9c305f5f615 2024-12-06T06:15:21,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/fe885255287d49258c94a4982b053487 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/fe885255287d49258c94a4982b053487 2024-12-06T06:15:21,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/955e6d205a03459c93654495336e3231 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/955e6d205a03459c93654495336e3231 2024-12-06T06:15:21,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1d12373c9555420681b390a20d11dbaf to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1d12373c9555420681b390a20d11dbaf 2024-12-06T06:15:21,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1b2b29bf8ff24bd2b70506be21cb5c4b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/1b2b29bf8ff24bd2b70506be21cb5c4b 2024-12-06T06:15:21,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/f289f94985cb4c41ae60f0804ffc8b8f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/f289f94985cb4c41ae60f0804ffc8b8f 2024-12-06T06:15:21,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/26c399a013fe482b946ce0a783aab757 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/26c399a013fe482b946ce0a783aab757 2024-12-06T06:15:21,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/7095f591977d4e4ea3fc7001c37376e3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/7095f591977d4e4ea3fc7001c37376e3 2024-12-06T06:15:21,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6e0d4dc7245a4451b0015915354bd2fb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/6e0d4dc7245a4451b0015915354bd2fb 2024-12-06T06:15:21,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/9fa099575ba441c49e2eeeee14c02455 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/9fa099575ba441c49e2eeeee14c02455 2024-12-06T06:15:21,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/22352606b6d049078f36ef26df9250fc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/22352606b6d049078f36ef26df9250fc 2024-12-06T06:15:21,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d80baf1e82fd47328c4abeb579a0b97b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/d80baf1e82fd47328c4abeb579a0b97b 2024-12-06T06:15:21,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/c49f2dafec574179985346b09ecf0815 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/c49f2dafec574179985346b09ecf0815 2024-12-06T06:15:21,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/2f97a525a69a4a5a9e2ee71a1a17da33 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/2f97a525a69a4a5a9e2ee71a1a17da33 2024-12-06T06:15:21,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/582024b1619e430d9eb073775d433671 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/582024b1619e430d9eb073775d433671 2024-12-06T06:15:21,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/49005dad29384614ad15fd7dad8977d1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/49005dad29384614ad15fd7dad8977d1 2024-12-06T06:15:21,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/14dd9ec2988841f29cd7965c4cb6dca4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bf749691e4004a7880db59fc219dbcdb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6fc7c7b51d414bd0b26c382d08c47006, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d78f1c6d0bc7450ab436f19b7d7cbcf0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1ec346784cf04cab9bcd6ff837d81ed6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49e30c8b60824c8f9a62cc6b12acf307, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/7fb5cb81e3b34a939bf3e7e988790a00, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1cb6c87cff6f4e37ae1510b3a88d8a99, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8304144f81b147d8b190846ab7abd473, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eeae7c5f82b343da9d56a5691cdfc95c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eb6a51bd9d9d4e10a4317b864f8c3ebc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/66c04aaf2ccc4936993278a8b3daade8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/80a3063ac0e245e1af13cd86c1e8d5b5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/16a372703a894603b423dbddce690c2d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d7f493435ff9469ca169eccf443f1acf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/b4bf4c8b61004d33a42929dde123237a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e398d35400f94343aa0a121f338edaa7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/610ab54b133540bcab14e170308efeb0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/2eaf680df6dc4e779cb3ed31d0041350, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bfcb62dc9fe0421bb4fe4eafe43d57d9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a948356fc961442b8baaac314c0cd708, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/dcb5c180e5344836bafb5eeb01b65c65, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/85743109d1d24c0cb1db343436645a65, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a59c04406b5946fda7d1f0ea20411528, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8aa664ce4be44930a6ec70d7fb247231, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/765ff72e19c04f47a8d68e93cc1463be, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/41b799d3f2414084aa1e680aff5262ac, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/3876969e69534eb390cd6015b1686adc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/270c830392c946a5ac4bd2b45c6207c9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6a8d9a0cda2e4c7893db50f1edd623f3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6d6a82131e884087b5b20be066c2896a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49a42e8e23d6408a8edc7de5f685d9ed, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/f7ac5e3bd52342a69b9bd6b34fd26101, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/0aa97632686c41109a48074eb5059f4a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e5ebd39fec1342c4a27b4e6e81715cdf] to archive 2024-12-06T06:15:21,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:21,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/14dd9ec2988841f29cd7965c4cb6dca4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/14dd9ec2988841f29cd7965c4cb6dca4 2024-12-06T06:15:21,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bf749691e4004a7880db59fc219dbcdb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bf749691e4004a7880db59fc219dbcdb 2024-12-06T06:15:21,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6fc7c7b51d414bd0b26c382d08c47006 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6fc7c7b51d414bd0b26c382d08c47006 2024-12-06T06:15:21,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d78f1c6d0bc7450ab436f19b7d7cbcf0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d78f1c6d0bc7450ab436f19b7d7cbcf0 2024-12-06T06:15:21,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1ec346784cf04cab9bcd6ff837d81ed6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1ec346784cf04cab9bcd6ff837d81ed6 2024-12-06T06:15:21,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49e30c8b60824c8f9a62cc6b12acf307 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49e30c8b60824c8f9a62cc6b12acf307 2024-12-06T06:15:21,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/7fb5cb81e3b34a939bf3e7e988790a00 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/7fb5cb81e3b34a939bf3e7e988790a00 2024-12-06T06:15:21,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1cb6c87cff6f4e37ae1510b3a88d8a99 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/1cb6c87cff6f4e37ae1510b3a88d8a99 2024-12-06T06:15:21,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8304144f81b147d8b190846ab7abd473 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8304144f81b147d8b190846ab7abd473 2024-12-06T06:15:21,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eeae7c5f82b343da9d56a5691cdfc95c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eeae7c5f82b343da9d56a5691cdfc95c 2024-12-06T06:15:21,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eb6a51bd9d9d4e10a4317b864f8c3ebc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/eb6a51bd9d9d4e10a4317b864f8c3ebc 2024-12-06T06:15:21,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/66c04aaf2ccc4936993278a8b3daade8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/66c04aaf2ccc4936993278a8b3daade8 2024-12-06T06:15:21,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/80a3063ac0e245e1af13cd86c1e8d5b5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/80a3063ac0e245e1af13cd86c1e8d5b5 2024-12-06T06:15:21,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/16a372703a894603b423dbddce690c2d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/16a372703a894603b423dbddce690c2d 2024-12-06T06:15:21,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d7f493435ff9469ca169eccf443f1acf to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/d7f493435ff9469ca169eccf443f1acf 2024-12-06T06:15:21,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/b4bf4c8b61004d33a42929dde123237a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/b4bf4c8b61004d33a42929dde123237a 2024-12-06T06:15:21,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e398d35400f94343aa0a121f338edaa7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e398d35400f94343aa0a121f338edaa7 2024-12-06T06:15:21,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/610ab54b133540bcab14e170308efeb0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/610ab54b133540bcab14e170308efeb0 2024-12-06T06:15:21,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/2eaf680df6dc4e779cb3ed31d0041350 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/2eaf680df6dc4e779cb3ed31d0041350 2024-12-06T06:15:21,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bfcb62dc9fe0421bb4fe4eafe43d57d9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/bfcb62dc9fe0421bb4fe4eafe43d57d9 2024-12-06T06:15:21,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a948356fc961442b8baaac314c0cd708 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a948356fc961442b8baaac314c0cd708 2024-12-06T06:15:21,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/dcb5c180e5344836bafb5eeb01b65c65 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/dcb5c180e5344836bafb5eeb01b65c65 2024-12-06T06:15:21,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/85743109d1d24c0cb1db343436645a65 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/85743109d1d24c0cb1db343436645a65 2024-12-06T06:15:21,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a59c04406b5946fda7d1f0ea20411528 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/a59c04406b5946fda7d1f0ea20411528 2024-12-06T06:15:21,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8aa664ce4be44930a6ec70d7fb247231 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/8aa664ce4be44930a6ec70d7fb247231 2024-12-06T06:15:21,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/765ff72e19c04f47a8d68e93cc1463be to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/765ff72e19c04f47a8d68e93cc1463be 2024-12-06T06:15:21,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/41b799d3f2414084aa1e680aff5262ac to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/41b799d3f2414084aa1e680aff5262ac 2024-12-06T06:15:21,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/3876969e69534eb390cd6015b1686adc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/3876969e69534eb390cd6015b1686adc 2024-12-06T06:15:21,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/270c830392c946a5ac4bd2b45c6207c9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/270c830392c946a5ac4bd2b45c6207c9 2024-12-06T06:15:21,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6a8d9a0cda2e4c7893db50f1edd623f3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6a8d9a0cda2e4c7893db50f1edd623f3 2024-12-06T06:15:21,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6d6a82131e884087b5b20be066c2896a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/6d6a82131e884087b5b20be066c2896a 2024-12-06T06:15:21,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49a42e8e23d6408a8edc7de5f685d9ed to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/49a42e8e23d6408a8edc7de5f685d9ed 2024-12-06T06:15:21,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/f7ac5e3bd52342a69b9bd6b34fd26101 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/f7ac5e3bd52342a69b9bd6b34fd26101 2024-12-06T06:15:21,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/0aa97632686c41109a48074eb5059f4a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/0aa97632686c41109a48074eb5059f4a 2024-12-06T06:15:21,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e5ebd39fec1342c4a27b4e6e81715cdf to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e5ebd39fec1342c4a27b4e6e81715cdf 2024-12-06T06:15:21,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a3806ac44be14c7f8c514f7fd3a5c9f7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/cb7bebec57fc43ca804cdc3d24b14b91, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/749dcd5011174ef6842601fc0ff2bfc8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/bad29907c76242e9b9884c31d4b6af4d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/636840599cdd4a868f52a5cc5f394361, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ee78d4ea18724de3a290f886c2202f43, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/2770e174a9404e6fba768f1f50d383d0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/9508f3f1135244b78bfc1cc762ca82c1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/44159779f56942d7b540013ed42b1584, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ff8d9450c2a24eafb0e8c5c07fe496cd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/771724ae62a840158646fb920791889a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6d0100ebbed644aca7752b0ac3d9cb96, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d7cf5230f0ae46c4be31aa9a96d54865, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/49ab7923e38942e88d1e468b6537643a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/82856fd8c5c240f8842e726072cc1aa9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6dfad5bfb5da4b8a90c6988a0cd430f6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d33a30468c2a4ec8ac2e7ab42a76619a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/62f2c81a221e423ca4fa0ed70bbe65d1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e5fdd4f7c237479994ad1e247471e3dd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/0849bdff0e6a40d6bf4969cf6e88c1c4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b9c395ed9d674d628a79f4ab37444852, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/df57cd745a094d58bb043ed250038371, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/07a28dbff0d2495aaaa90ccefb4a9e0f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/5ffdb77fcd9a4bab8108bc6327e6a637, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/27fddb338c134b5ea79b88e5e9ad766c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/f50e20ae67ac41f39ca09e45ecaa8507, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/28136cf534224de38af6da186ba62042, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/38030f77750f420b96699466d0860574, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a42b00e716f84fbda1e1f3d61c7b52cd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/eba417caca3e4787838a74bda123db81, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a61d9520d5054824a9503cda0b434645, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e885e7e14d1b431e91bbe0d3910ef91a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ed10c57ad782489aadf407a60608b0ae, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d245d41d42664bec9711bcacf43452b2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b6c73e1619c7490ab2d60f6672d7ed36] to archive 2024-12-06T06:15:21,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:21,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a3806ac44be14c7f8c514f7fd3a5c9f7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a3806ac44be14c7f8c514f7fd3a5c9f7 2024-12-06T06:15:21,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/cb7bebec57fc43ca804cdc3d24b14b91 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/cb7bebec57fc43ca804cdc3d24b14b91 2024-12-06T06:15:21,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/749dcd5011174ef6842601fc0ff2bfc8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/749dcd5011174ef6842601fc0ff2bfc8 2024-12-06T06:15:21,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/bad29907c76242e9b9884c31d4b6af4d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/bad29907c76242e9b9884c31d4b6af4d 2024-12-06T06:15:21,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/636840599cdd4a868f52a5cc5f394361 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/636840599cdd4a868f52a5cc5f394361 2024-12-06T06:15:21,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ee78d4ea18724de3a290f886c2202f43 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ee78d4ea18724de3a290f886c2202f43 2024-12-06T06:15:21,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/2770e174a9404e6fba768f1f50d383d0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/2770e174a9404e6fba768f1f50d383d0 2024-12-06T06:15:21,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/9508f3f1135244b78bfc1cc762ca82c1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/9508f3f1135244b78bfc1cc762ca82c1 2024-12-06T06:15:21,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/44159779f56942d7b540013ed42b1584 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/44159779f56942d7b540013ed42b1584 2024-12-06T06:15:21,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ff8d9450c2a24eafb0e8c5c07fe496cd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ff8d9450c2a24eafb0e8c5c07fe496cd 2024-12-06T06:15:21,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/771724ae62a840158646fb920791889a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/771724ae62a840158646fb920791889a 2024-12-06T06:15:21,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6d0100ebbed644aca7752b0ac3d9cb96 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6d0100ebbed644aca7752b0ac3d9cb96 2024-12-06T06:15:21,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d7cf5230f0ae46c4be31aa9a96d54865 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d7cf5230f0ae46c4be31aa9a96d54865 2024-12-06T06:15:21,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/49ab7923e38942e88d1e468b6537643a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/49ab7923e38942e88d1e468b6537643a 2024-12-06T06:15:21,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/82856fd8c5c240f8842e726072cc1aa9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/82856fd8c5c240f8842e726072cc1aa9 2024-12-06T06:15:21,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6dfad5bfb5da4b8a90c6988a0cd430f6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/6dfad5bfb5da4b8a90c6988a0cd430f6 2024-12-06T06:15:21,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d33a30468c2a4ec8ac2e7ab42a76619a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d33a30468c2a4ec8ac2e7ab42a76619a 2024-12-06T06:15:21,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/62f2c81a221e423ca4fa0ed70bbe65d1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/62f2c81a221e423ca4fa0ed70bbe65d1 2024-12-06T06:15:21,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e5fdd4f7c237479994ad1e247471e3dd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e5fdd4f7c237479994ad1e247471e3dd 2024-12-06T06:15:21,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/0849bdff0e6a40d6bf4969cf6e88c1c4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/0849bdff0e6a40d6bf4969cf6e88c1c4 2024-12-06T06:15:21,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b9c395ed9d674d628a79f4ab37444852 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b9c395ed9d674d628a79f4ab37444852 2024-12-06T06:15:21,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/df57cd745a094d58bb043ed250038371 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/df57cd745a094d58bb043ed250038371 2024-12-06T06:15:21,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/07a28dbff0d2495aaaa90ccefb4a9e0f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/07a28dbff0d2495aaaa90ccefb4a9e0f 2024-12-06T06:15:21,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/5ffdb77fcd9a4bab8108bc6327e6a637 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/5ffdb77fcd9a4bab8108bc6327e6a637 2024-12-06T06:15:21,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/27fddb338c134b5ea79b88e5e9ad766c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/27fddb338c134b5ea79b88e5e9ad766c 2024-12-06T06:15:21,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/f50e20ae67ac41f39ca09e45ecaa8507 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/f50e20ae67ac41f39ca09e45ecaa8507 2024-12-06T06:15:21,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/28136cf534224de38af6da186ba62042 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/28136cf534224de38af6da186ba62042 2024-12-06T06:15:21,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/38030f77750f420b96699466d0860574 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/38030f77750f420b96699466d0860574 2024-12-06T06:15:21,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a42b00e716f84fbda1e1f3d61c7b52cd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a42b00e716f84fbda1e1f3d61c7b52cd 2024-12-06T06:15:21,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/eba417caca3e4787838a74bda123db81 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/eba417caca3e4787838a74bda123db81 2024-12-06T06:15:21,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a61d9520d5054824a9503cda0b434645 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/a61d9520d5054824a9503cda0b434645 2024-12-06T06:15:21,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e885e7e14d1b431e91bbe0d3910ef91a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/e885e7e14d1b431e91bbe0d3910ef91a 2024-12-06T06:15:21,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ed10c57ad782489aadf407a60608b0ae to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/ed10c57ad782489aadf407a60608b0ae 2024-12-06T06:15:21,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d245d41d42664bec9711bcacf43452b2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/d245d41d42664bec9711bcacf43452b2 2024-12-06T06:15:21,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b6c73e1619c7490ab2d60f6672d7ed36 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/b6c73e1619c7490ab2d60f6672d7ed36 2024-12-06T06:15:21,559 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/recovered.edits/464.seqid, newMaxSeqId=464, maxSeqId=1 2024-12-06T06:15:21,560 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. 2024-12-06T06:15:21,560 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for aa1db8fd546545da53b24cabfd6bfcf1: Waiting for close lock at 1733465720227Running coprocessor pre-close hooks at 1733465720227Disabling compacts and flushes for region at 1733465720227Disabling writes for close at 1733465720227Obtaining lock to block concurrent updates at 1733465720227Preparing flush snapshotting stores in aa1db8fd546545da53b24cabfd6bfcf1 at 1733465720227Finished memstore snapshotting TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733465720228 (+1 ms)Flushing stores of TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1. at 1733465720228Flushing aa1db8fd546545da53b24cabfd6bfcf1/A: creating writer at 1733465720228Flushing aa1db8fd546545da53b24cabfd6bfcf1/A: appending metadata at 1733465720231 (+3 ms)Flushing aa1db8fd546545da53b24cabfd6bfcf1/A: closing flushed file at 1733465720231Flushing aa1db8fd546545da53b24cabfd6bfcf1/B: creating writer at 1733465720639 (+408 ms)Flushing aa1db8fd546545da53b24cabfd6bfcf1/B: appending metadata at 1733465720642 (+3 ms)Flushing aa1db8fd546545da53b24cabfd6bfcf1/B: closing flushed file at 1733465720642Flushing aa1db8fd546545da53b24cabfd6bfcf1/C: creating writer at 1733465721051 (+409 ms)Flushing aa1db8fd546545da53b24cabfd6bfcf1/C: appending metadata at 1733465721053 (+2 ms)Flushing aa1db8fd546545da53b24cabfd6bfcf1/C: closing flushed file at 1733465721054 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@377b9092: reopening flushed file at 1733465721461 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b4efe42: reopening flushed file at 1733465721464 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@345b9616: reopening flushed file at 1733465721467 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for aa1db8fd546545da53b24cabfd6bfcf1 in 1244ms, sequenceid=461, compaction requested=false at 1733465721471 (+4 ms)Writing region close event to WAL at 1733465721557 (+86 ms)Running coprocessor post-close hooks at 1733465721560 (+3 ms)Closed at 1733465721560 2024-12-06T06:15:21,561 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:21,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=aa1db8fd546545da53b24cabfd6bfcf1, regionState=CLOSED 2024-12-06T06:15:21,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:15:21,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-06T06:15:21,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure aa1db8fd546545da53b24cabfd6bfcf1, server=6336e8474e1c,39987,1733465620995 in 1.4890 sec 2024-12-06T06:15:21,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-06T06:15:21,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa1db8fd546545da53b24cabfd6bfcf1, UNASSIGN in 1.4930 sec 2024-12-06T06:15:21,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-06T06:15:21,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4960 sec 2024-12-06T06:15:21,569 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465721569"}]},"ts":"1733465721569"} 2024-12-06T06:15:21,570 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T06:15:21,570 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T06:15:21,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5050 sec 2024-12-06T06:15:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-06T06:15:22,214 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:22,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T06:15:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:22,216 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-06T06:15:22,217 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:22,219 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:22,221 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/recovered.edits] 2024-12-06T06:15:22,223 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b526368ce79140b9b39d5a23553fa635 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/b526368ce79140b9b39d5a23553fa635 2024-12-06T06:15:22,224 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/baf5a63b28974ad187ee2f76493d9521 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/A/baf5a63b28974ad187ee2f76493d9521 2024-12-06T06:15:22,226 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/99b4f1e74e1d472cb4ecc223d6d82308 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/99b4f1e74e1d472cb4ecc223d6d82308 2024-12-06T06:15:22,227 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e85e7e0ac1fd46709d1c45022f8d422b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/B/e85e7e0ac1fd46709d1c45022f8d422b 2024-12-06T06:15:22,229 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/450bbbd1969349db94aa71c7519d6703 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/450bbbd1969349db94aa71c7519d6703 2024-12-06T06:15:22,230 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/4c25e8fe0a104b4eb9b9941f8afff737 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/C/4c25e8fe0a104b4eb9b9941f8afff737 2024-12-06T06:15:22,233 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/recovered.edits/464.seqid to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1/recovered.edits/464.seqid 2024-12-06T06:15:22,233 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/aa1db8fd546545da53b24cabfd6bfcf1 2024-12-06T06:15:22,233 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T06:15:22,235 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:22,237 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T06:15:22,239 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T06:15:22,240 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:22,240 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T06:15:22,240 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733465722240"}]},"ts":"9223372036854775807"} 2024-12-06T06:15:22,242 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T06:15:22,242 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aa1db8fd546545da53b24cabfd6bfcf1, NAME => 'TestAcidGuarantees,,1733465689255.aa1db8fd546545da53b24cabfd6bfcf1.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T06:15:22,242 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T06:15:22,242 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733465722242"}]},"ts":"9223372036854775807"} 2024-12-06T06:15:22,244 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T06:15:22,244 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:22,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-12-06T06:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-06T06:15:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T06:15:22,324 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:22,333 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 435), ProcessCount=11 (was 11), AvailableMemoryMB=4826 (was 5077) 2024-12-06T06:15:22,341 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=4825 2024-12-06T06:15:22,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:15:22,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:15:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:22,345 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T06:15:22,345 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:22,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-12-06T06:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T06:15:22,346 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T06:15:22,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742221_1397 (size=1061) 2024-12-06T06:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T06:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T06:15:22,752 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea 2024-12-06T06:15:22,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742222_1398 (size=53) 2024-12-06T06:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T06:15:23,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:15:23,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9d97d8dc224d0a067fc9941f0b9f555b, disabling compactions & flushes 2024-12-06T06:15:23,157 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:23,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:23,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. after waiting 0 ms 2024-12-06T06:15:23,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:23,157 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:23,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d97d8dc224d0a067fc9941f0b9f555b: Waiting for close lock at 1733465723157Disabling compacts and flushes for region at 1733465723157Disabling writes for close at 1733465723157Writing region close event to WAL at 1733465723157Closed at 1733465723157 2024-12-06T06:15:23,158 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T06:15:23,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733465723158"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733465723158"}]},"ts":"1733465723158"} 2024-12-06T06:15:23,160 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T06:15:23,161 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T06:15:23,161 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465723161"}]},"ts":"1733465723161"} 2024-12-06T06:15:23,162 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T06:15:23,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, ASSIGN}] 2024-12-06T06:15:23,163 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, ASSIGN 2024-12-06T06:15:23,163 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, ASSIGN; state=OFFLINE, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=false 2024-12-06T06:15:23,314 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9d97d8dc224d0a067fc9941f0b9f555b, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:23,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, ASSIGN because future has completed 2024-12-06T06:15:23,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:15:23,471 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:23,471 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:15:23,471 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,471 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:15:23,471 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,471 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,472 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,473 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:23,473 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d97d8dc224d0a067fc9941f0b9f555b columnFamilyName A 2024-12-06T06:15:23,474 DEBUG [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:23,474 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(327): Store=9d97d8dc224d0a067fc9941f0b9f555b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:23,474 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,475 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:23,475 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d97d8dc224d0a067fc9941f0b9f555b columnFamilyName B 2024-12-06T06:15:23,475 DEBUG [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:23,475 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(327): Store=9d97d8dc224d0a067fc9941f0b9f555b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:23,475 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,476 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:23,476 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d97d8dc224d0a067fc9941f0b9f555b columnFamilyName C 2024-12-06T06:15:23,476 DEBUG [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:23,476 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(327): Store=9d97d8dc224d0a067fc9941f0b9f555b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:23,476 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:23,476 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,477 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,477 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,478 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,478 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,478 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:15:23,479 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,480 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T06:15:23,481 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 9d97d8dc224d0a067fc9941f0b9f555b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65788168, jitterRate=-0.019679903984069824}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:15:23,481 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:23,481 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 9d97d8dc224d0a067fc9941f0b9f555b: Running coprocessor pre-open hook at 1733465723471Writing region info on filesystem at 1733465723471Initializing all the Stores at 1733465723472 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465723472Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465723472Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465723472Cleaning up temporary data from old regions at 1733465723478 (+6 ms)Running coprocessor post-open hooks at 1733465723481 (+3 ms)Region opened successfully at 1733465723481 2024-12-06T06:15:23,482 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., pid=95, masterSystemTime=1733465723468 2024-12-06T06:15:23,483 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:23,483 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T06:15:23,484 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9d97d8dc224d0a067fc9941f0b9f555b, regionState=OPEN, openSeqNum=2, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:23,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:15:23,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-06T06:15:23,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 in 170 msec 2024-12-06T06:15:23,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-06T06:15:23,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, ASSIGN in 325 msec 2024-12-06T06:15:23,489 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T06:15:23,489 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465723489"}]},"ts":"1733465723489"} 2024-12-06T06:15:23,490 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T06:15:23,491 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T06:15:23,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-12-06T06:15:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T06:15:24,494 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:24,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:24,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:24,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:24,497 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a837f43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:24,498 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea4ebf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:24,499 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:24,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:24,501 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T06:15:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6336e8474e1c,40959,1733465620259 2024-12-06T06:15:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16d9ede5 2024-12-06T06:15:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T06:15:24,503 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T06:15:24,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:15:24,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:15:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:24,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742223_1399 (size=1097) 2024-12-06T06:15:24,914 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-06T06:15:24,914 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-06T06:15:24,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:15:24,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, REOPEN/MOVE}] 2024-12-06T06:15:24,919 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, REOPEN/MOVE 2024-12-06T06:15:24,919 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=9d97d8dc224d0a067fc9941f0b9f555b, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:24,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, REOPEN/MOVE because future has completed 2024-12-06T06:15:24,921 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:15:24,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:15:25,073 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,073 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:15:25,073 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 9d97d8dc224d0a067fc9941f0b9f555b, disabling compactions & flushes 2024-12-06T06:15:25,073 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,073 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,073 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. after waiting 0 ms 2024-12-06T06:15:25,073 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,076 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T06:15:25,077 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,077 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 9d97d8dc224d0a067fc9941f0b9f555b: Waiting for close lock at 1733465725073Running coprocessor pre-close hooks at 1733465725073Disabling compacts and flushes for region at 1733465725073Disabling writes for close at 1733465725073Writing region close event to WAL at 1733465725073Running coprocessor post-close hooks at 1733465725077 (+4 ms)Closed at 1733465725077 2024-12-06T06:15:25,077 WARN [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 9d97d8dc224d0a067fc9941f0b9f555b to self. 2024-12-06T06:15:25,079 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,079 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=9d97d8dc224d0a067fc9941f0b9f555b, regionState=CLOSED 2024-12-06T06:15:25,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:15:25,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-12-06T06:15:25,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 in 160 msec 2024-12-06T06:15:25,083 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, REOPEN/MOVE; state=CLOSED, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=true 2024-12-06T06:15:25,233 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=9d97d8dc224d0a067fc9941f0b9f555b, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, REOPEN/MOVE because future has completed 2024-12-06T06:15:25,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:15:25,389 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,389 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:15:25,390 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,390 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:15:25,390 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,390 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,391 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,392 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:25,392 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d97d8dc224d0a067fc9941f0b9f555b columnFamilyName A 2024-12-06T06:15:25,393 DEBUG [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:25,393 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(327): Store=9d97d8dc224d0a067fc9941f0b9f555b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:25,393 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,394 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:25,394 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d97d8dc224d0a067fc9941f0b9f555b columnFamilyName B 2024-12-06T06:15:25,394 DEBUG [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:25,394 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(327): Store=9d97d8dc224d0a067fc9941f0b9f555b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:25,394 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,395 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:25,395 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d97d8dc224d0a067fc9941f0b9f555b columnFamilyName C 2024-12-06T06:15:25,395 DEBUG [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:25,395 INFO [StoreOpener-9d97d8dc224d0a067fc9941f0b9f555b-1 {}] regionserver.HStore(327): Store=9d97d8dc224d0a067fc9941f0b9f555b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:25,395 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,395 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,396 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,396 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,397 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,397 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,398 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:15:25,398 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,399 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 9d97d8dc224d0a067fc9941f0b9f555b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66115812, jitterRate=-0.014797627925872803}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:15:25,399 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,400 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 9d97d8dc224d0a067fc9941f0b9f555b: Running coprocessor pre-open hook at 1733465725390Writing region info on filesystem at 1733465725390Initializing all the Stores at 1733465725390Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465725390Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465725391 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465725391Cleaning up temporary data from old regions at 1733465725397 (+6 ms)Running coprocessor post-open hooks at 1733465725399 (+2 ms)Region opened successfully at 1733465725399 2024-12-06T06:15:25,400 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., pid=100, masterSystemTime=1733465725386 2024-12-06T06:15:25,402 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,402 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=9d97d8dc224d0a067fc9941f0b9f555b, regionState=OPEN, openSeqNum=5, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:15:25,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-12-06T06:15:25,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 in 169 msec 2024-12-06T06:15:25,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-06T06:15:25,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, REOPEN/MOVE in 487 msec 2024-12-06T06:15:25,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-06T06:15:25,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-12-06T06:15:25,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-12-06T06:15:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-12-06T06:15:25,411 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:25,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,412 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8d0066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6521af3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,415 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e07b096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@737f1f28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd2e9ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aec7cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,421 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241c132e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,422 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c026d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,423 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60364f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c682dae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,426 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e323b6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41053b1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,429 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42467f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,430 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7905704b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,431 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c44f0ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,433 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d5c242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b7b919, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,434 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e521bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbcab87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dacd8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:25,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:25,437 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69246f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:25,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c370944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:25,439 DEBUG [Thread-1799 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:25,439 DEBUG [Thread-1792 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,439 DEBUG [Thread-1796 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,440 DEBUG [Thread-1805 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,440 DEBUG [Thread-1803 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,440 DEBUG [Thread-1794 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-06T06:15:25,440 DEBUG [Thread-1790 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,441 DEBUG [Thread-1803 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,441 DEBUG [Thread-1803 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T06:15:25,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,443 DEBUG [Thread-1807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,443 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:25,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,443 DEBUG [Thread-1788 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,444 DEBUG [Thread-1801 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,444 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:25,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:25,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:15:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:15:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:25,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ae092c2648af490f95b33a15d7823b62_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_1/A:col10/1733465725454/Put/seqid=0 2024-12-06T06:15:25,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742224_1400 (size=9714) 2024-12-06T06:15:25,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465725491, max timestamp=1733465725506Num uniques 859; 2024-12-06T06:15:25,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465725491, max timestamp=1733465725506Num uniques 859; 2024-12-06T06:15:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465785505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:15:25,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:15:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465725491, max timestamp=1733465725506Num uniques 859; 2024-12-06T06:15:25,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:15:25,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465785538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465785550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465785550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T06:15:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465785551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T06:15:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465785614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465785653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465785654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465785654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465785674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T06:15:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T06:15:25,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465785833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465785864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465785864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465785864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465785883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:25,899 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:25,903 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ae092c2648af490f95b33a15d7823b62_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ae092c2648af490f95b33a15d7823b62_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:25,904 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/5188018fb5a04e13af7bdffea1636953, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:25,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/5188018fb5a04e13af7bdffea1636953 is 175, key is test_row_1/A:col10/1733465725454/Put/seqid=0 2024-12-06T06:15:25,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T06:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:25,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:25,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742225_1401 (size=22361) 2024-12-06T06:15:25,915 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/5188018fb5a04e13af7bdffea1636953 2024-12-06T06:15:25,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/924ea17898624e3a801a22c759b25974 is 50, key is test_row_1/B:col10/1733465725454/Put/seqid=0 2024-12-06T06:15:25,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742226_1402 (size=9657) 2024-12-06T06:15:25,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/924ea17898624e3a801a22c759b25974 2024-12-06T06:15:25,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/a4e6822156534a33a9ad69cdb31cf103 is 50, key is test_row_1/C:col10/1733465725454/Put/seqid=0 2024-12-06T06:15:25,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742227_1403 (size=9657) 2024-12-06T06:15:26,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T06:15:26,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:26,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:26,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:26,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T06:15:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465786154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465786183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465786184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465786184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465786204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T06:15:26,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:26,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:26,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:26,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T06:15:26,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:26,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:26,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:26,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:26,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/a4e6822156534a33a9ad69cdb31cf103 2024-12-06T06:15:26,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/5188018fb5a04e13af7bdffea1636953 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/5188018fb5a04e13af7bdffea1636953 2024-12-06T06:15:26,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/5188018fb5a04e13af7bdffea1636953, entries=100, sequenceid=14, filesize=21.8 K 2024-12-06T06:15:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/924ea17898624e3a801a22c759b25974 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/924ea17898624e3a801a22c759b25974 2024-12-06T06:15:26,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/924ea17898624e3a801a22c759b25974, entries=100, sequenceid=14, filesize=9.4 K 2024-12-06T06:15:26,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/a4e6822156534a33a9ad69cdb31cf103 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/a4e6822156534a33a9ad69cdb31cf103 2024-12-06T06:15:26,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/a4e6822156534a33a9ad69cdb31cf103, entries=100, sequenceid=14, filesize=9.4 K 2024-12-06T06:15:26,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 9d97d8dc224d0a067fc9941f0b9f555b in 935ms, sequenceid=14, compaction requested=false 2024-12-06T06:15:26,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:26,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-06T06:15:26,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:26,522 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-06T06:15:26,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:26,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:26,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:26,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:26,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:26,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412069b8e868bf70f4a3aa4c74da577d99bd6_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465725533/Put/seqid=0 2024-12-06T06:15:26,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742228_1404 (size=12154) 2024-12-06T06:15:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T06:15:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:26,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465786704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465786704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465786707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465786715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465786723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465786824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465786833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:26,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:26,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:26,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:26,941 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412069b8e868bf70f4a3aa4c74da577d99bd6_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069b8e868bf70f4a3aa4c74da577d99bd6_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:26,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/260cdeb519dc4edca9cdcef66e1f2277, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/260cdeb519dc4edca9cdcef66e1f2277 is 175, key is test_row_0/A:col10/1733465725533/Put/seqid=0 2024-12-06T06:15:26,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742229_1405 (size=30955) 2024-12-06T06:15:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465787044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465787044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:27,085 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T06:15:27,352 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/260cdeb519dc4edca9cdcef66e1f2277 2024-12-06T06:15:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/08dd7d9a00824d6490ecd5391e121f38 is 50, key is test_row_0/B:col10/1733465725533/Put/seqid=0 2024-12-06T06:15:27,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742230_1406 (size=12001) 2024-12-06T06:15:27,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465787364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:27,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465787364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T06:15:27,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465787723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:27,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465787724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:27,736 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:27,736 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:27,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465787744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:27,753 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:27,767 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/08dd7d9a00824d6490ecd5391e121f38 2024-12-06T06:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/25a31716541c42e6b60c82599ca0ace8 is 50, key is test_row_0/C:col10/1733465725533/Put/seqid=0 2024-12-06T06:15:27,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742231_1407 (size=12001) 2024-12-06T06:15:27,778 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/25a31716541c42e6b60c82599ca0ace8 2024-12-06T06:15:27,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/260cdeb519dc4edca9cdcef66e1f2277 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/260cdeb519dc4edca9cdcef66e1f2277 2024-12-06T06:15:27,785 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/260cdeb519dc4edca9cdcef66e1f2277, entries=150, sequenceid=50, filesize=30.2 K 2024-12-06T06:15:27,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/08dd7d9a00824d6490ecd5391e121f38 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/08dd7d9a00824d6490ecd5391e121f38 2024-12-06T06:15:27,789 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/08dd7d9a00824d6490ecd5391e121f38, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T06:15:27,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/25a31716541c42e6b60c82599ca0ace8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/25a31716541c42e6b60c82599ca0ace8 2024-12-06T06:15:27,793 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/25a31716541c42e6b60c82599ca0ace8, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T06:15:27,793 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 9d97d8dc224d0a067fc9941f0b9f555b in 1271ms, sequenceid=50, compaction requested=false 2024-12-06T06:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-06T06:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-06T06:15:27,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-06T06:15:27,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3510 sec 2024-12-06T06:15:27,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3590 sec 2024-12-06T06:15:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:27,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:15:27,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:27,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:27,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:27,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:27,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:27,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:27,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120699b658054f3d4402925396b02edf026b_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465726698/Put/seqid=0 2024-12-06T06:15:27,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742232_1408 (size=14594) 2024-12-06T06:15:27,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465727964, max timestamp=1733465727984Num uniques 859; 2024-12-06T06:15:27,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465727964, max timestamp=1733465727984Num uniques 859; 2024-12-06T06:15:27,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:27,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:27,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465727964, max timestamp=1733465727984Num uniques 859; 2024-12-06T06:15:27,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:28,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465788073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:28,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465788082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:28,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465788193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:28,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465788193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:28,310 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:28,313 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120699b658054f3d4402925396b02edf026b_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120699b658054f3d4402925396b02edf026b_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:28,314 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/fde34abcd7a4426386b504005e0901e7, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:28,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/fde34abcd7a4426386b504005e0901e7 is 175, key is test_row_0/A:col10/1733465726698/Put/seqid=0 2024-12-06T06:15:28,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742233_1409 (size=39549) 2024-12-06T06:15:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465788414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465788414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:28,718 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/fde34abcd7a4426386b504005e0901e7 2024-12-06T06:15:28,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/0a6837b5b6b44f3288ba002909e2bd9f is 50, key is test_row_0/B:col10/1733465726698/Put/seqid=0 2024-12-06T06:15:28,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742234_1410 (size=12001) 2024-12-06T06:15:28,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465788734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:28,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465788734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/0a6837b5b6b44f3288ba002909e2bd9f 2024-12-06T06:15:29,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8e5b0d03b86c42608e207e70a7c6ce6d is 50, key is test_row_0/C:col10/1733465726698/Put/seqid=0 2024-12-06T06:15:29,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742235_1411 (size=12001) 2024-12-06T06:15:29,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8e5b0d03b86c42608e207e70a7c6ce6d 2024-12-06T06:15:29,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/fde34abcd7a4426386b504005e0901e7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/fde34abcd7a4426386b504005e0901e7 2024-12-06T06:15:29,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/fde34abcd7a4426386b504005e0901e7, entries=200, sequenceid=60, filesize=38.6 K 2024-12-06T06:15:29,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/0a6837b5b6b44f3288ba002909e2bd9f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/0a6837b5b6b44f3288ba002909e2bd9f 2024-12-06T06:15:29,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/0a6837b5b6b44f3288ba002909e2bd9f, entries=150, sequenceid=60, filesize=11.7 K 2024-12-06T06:15:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8e5b0d03b86c42608e207e70a7c6ce6d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e5b0d03b86c42608e207e70a7c6ce6d 2024-12-06T06:15:29,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e5b0d03b86c42608e207e70a7c6ce6d, entries=150, sequenceid=60, filesize=11.7 K 2024-12-06T06:15:29,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9d97d8dc224d0a067fc9941f0b9f555b in 1270ms, sequenceid=60, compaction requested=true 2024-12-06T06:15:29,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:29,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:29,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:29,160 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:29,160 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:29,161 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:29,161 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:29,161 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/A is initiating minor compaction (all files) 2024-12-06T06:15:29,161 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/B is initiating minor compaction (all files) 2024-12-06T06:15:29,161 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/A in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:29,161 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/B in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:29,161 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/5188018fb5a04e13af7bdffea1636953, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/260cdeb519dc4edca9cdcef66e1f2277, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/fde34abcd7a4426386b504005e0901e7] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=90.7 K 2024-12-06T06:15:29,161 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:29,161 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/924ea17898624e3a801a22c759b25974, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/08dd7d9a00824d6490ecd5391e121f38, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/0a6837b5b6b44f3288ba002909e2bd9f] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=32.9 K 2024-12-06T06:15:29,161 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/5188018fb5a04e13af7bdffea1636953, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/260cdeb519dc4edca9cdcef66e1f2277, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/fde34abcd7a4426386b504005e0901e7] 2024-12-06T06:15:29,162 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5188018fb5a04e13af7bdffea1636953, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465725450 2024-12-06T06:15:29,162 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 924ea17898624e3a801a22c759b25974, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465725450 2024-12-06T06:15:29,162 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 08dd7d9a00824d6490ecd5391e121f38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465725533 2024-12-06T06:15:29,162 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260cdeb519dc4edca9cdcef66e1f2277, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465725533 2024-12-06T06:15:29,162 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6837b5b6b44f3288ba002909e2bd9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465726698 2024-12-06T06:15:29,162 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde34abcd7a4426386b504005e0901e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465726698 2024-12-06T06:15:29,168 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:29,169 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:29,170 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/084309166d7e4535b61acfba19afd7d0 is 50, key is test_row_0/B:col10/1733465726698/Put/seqid=0 2024-12-06T06:15:29,170 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412060ffbfbe7aacf406f8eed5569c65e1ff7_9d97d8dc224d0a067fc9941f0b9f555b store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:29,172 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412060ffbfbe7aacf406f8eed5569c65e1ff7_9d97d8dc224d0a067fc9941f0b9f555b, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:29,173 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412060ffbfbe7aacf406f8eed5569c65e1ff7_9d97d8dc224d0a067fc9941f0b9f555b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:29,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742236_1412 (size=12104) 2024-12-06T06:15:29,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742237_1413 (size=4469) 2024-12-06T06:15:29,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-12-06T06:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:29,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:29,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206806997ed2297422596123c3832ccacba_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465728080/Put/seqid=0 2024-12-06T06:15:29,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742238_1414 (size=17034) 2024-12-06T06:15:29,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465789297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465789307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465789414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465789414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,583 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:29,583 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/084309166d7e4535b61acfba19afd7d0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/084309166d7e4535b61acfba19afd7d0 2024-12-06T06:15:29,583 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/1a1160b096034111bdf9da676ee4db6b is 175, key is test_row_0/A:col10/1733465726698/Put/seqid=0 2024-12-06T06:15:29,588 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/B of 9d97d8dc224d0a067fc9941f0b9f555b into 084309166d7e4535b61acfba19afd7d0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:29,588 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:29,588 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/B, priority=13, startTime=1733465729160; duration=0sec 2024-12-06T06:15:29,588 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:29,588 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:B 2024-12-06T06:15:29,588 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:29,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742239_1415 (size=31058) 2024-12-06T06:15:29,595 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:29,595 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/C is initiating minor compaction (all files) 2024-12-06T06:15:29,595 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/C in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:29,595 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/a4e6822156534a33a9ad69cdb31cf103, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/25a31716541c42e6b60c82599ca0ace8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e5b0d03b86c42608e207e70a7c6ce6d] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=32.9 K 2024-12-06T06:15:29,595 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a4e6822156534a33a9ad69cdb31cf103, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465725450 2024-12-06T06:15:29,596 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 25a31716541c42e6b60c82599ca0ace8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465725533 2024-12-06T06:15:29,596 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e5b0d03b86c42608e207e70a7c6ce6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465726698 2024-12-06T06:15:29,604 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/1a1160b096034111bdf9da676ee4db6b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/1a1160b096034111bdf9da676ee4db6b 2024-12-06T06:15:29,604 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#C#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:29,605 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/3bdcea62e4484b229deb773683bd256a is 50, key is test_row_0/C:col10/1733465726698/Put/seqid=0 2024-12-06T06:15:29,611 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/A of 9d97d8dc224d0a067fc9941f0b9f555b into 1a1160b096034111bdf9da676ee4db6b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:29,611 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:29,611 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/A, priority=13, startTime=1733465729160; duration=0sec 2024-12-06T06:15:29,611 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:29,611 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:A 2024-12-06T06:15:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-06T06:15:29,614 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:29,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-06T06:15:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T06:15:29,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742240_1416 (size=12104) 2024-12-06T06:15:29,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:29,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:29,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:29,626 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/3bdcea62e4484b229deb773683bd256a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/3bdcea62e4484b229deb773683bd256a 2024-12-06T06:15:29,631 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/C of 9d97d8dc224d0a067fc9941f0b9f555b into 3bdcea62e4484b229deb773683bd256a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:29,631 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:29,631 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/C, priority=13, startTime=1733465729161; duration=0sec 2024-12-06T06:15:29,632 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:29,632 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:C 2024-12-06T06:15:29,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465789623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465789623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,686 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:29,694 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206806997ed2297422596123c3832ccacba_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206806997ed2297422596123c3832ccacba_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:29,695 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/a9791c4bbbc34c1bb131dd7f0c2424b8, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:29,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/a9791c4bbbc34c1bb131dd7f0c2424b8 is 175, key is test_row_0/A:col10/1733465728080/Put/seqid=0 2024-12-06T06:15:29,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742241_1417 (size=48139) 2024-12-06T06:15:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T06:15:29,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465789754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,758 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465789764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:29,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:29,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:29,775 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:29,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465789773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,785 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:29,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:29,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T06:15:29,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465789943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:29,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465789943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:30,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:30,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:30,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,101 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/a9791c4bbbc34c1bb131dd7f0c2424b8 2024-12-06T06:15:30,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/8377328f08934d9d9ab93ef09bff1de8 is 50, key is test_row_0/B:col10/1733465728080/Put/seqid=0 2024-12-06T06:15:30,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742242_1418 (size=12001) 2024-12-06T06:15:30,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T06:15:30,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:30,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:30,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465790463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:30,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465790464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:30,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/8377328f08934d9d9ab93ef09bff1de8 2024-12-06T06:15:30,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/029e377109bc473181695d682fff929d is 50, key is test_row_0/C:col10/1733465728080/Put/seqid=0 2024-12-06T06:15:30,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742243_1419 (size=12001) 2024-12-06T06:15:30,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T06:15:30,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:30,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:30,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/029e377109bc473181695d682fff929d 2024-12-06T06:15:30,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/a9791c4bbbc34c1bb131dd7f0c2424b8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/a9791c4bbbc34c1bb131dd7f0c2424b8 2024-12-06T06:15:30,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/a9791c4bbbc34c1bb131dd7f0c2424b8, entries=250, sequenceid=99, filesize=47.0 K 2024-12-06T06:15:30,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/8377328f08934d9d9ab93ef09bff1de8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/8377328f08934d9d9ab93ef09bff1de8 2024-12-06T06:15:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/8377328f08934d9d9ab93ef09bff1de8, entries=150, sequenceid=99, filesize=11.7 K 2024-12-06T06:15:30,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/029e377109bc473181695d682fff929d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/029e377109bc473181695d682fff929d 2024-12-06T06:15:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/029e377109bc473181695d682fff929d, entries=150, sequenceid=99, filesize=11.7 K 2024-12-06T06:15:30,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for 9d97d8dc224d0a067fc9941f0b9f555b in 1676ms, sequenceid=99, compaction requested=false 2024-12-06T06:15:30,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-06T06:15:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:31,002 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T06:15:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d0f03865870b44aeb703b3dc2fe26bda_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465729306/Put/seqid=0 2024-12-06T06:15:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742244_1420 (size=9714) 2024-12-06T06:15:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,025 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d0f03865870b44aeb703b3dc2fe26bda_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d0f03865870b44aeb703b3dc2fe26bda_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/cce498b65b444ed1a3b7530ae8ec5d9c, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/cce498b65b444ed1a3b7530ae8ec5d9c is 175, key is test_row_0/A:col10/1733465729306/Put/seqid=0 2024-12-06T06:15:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742245_1421 (size=22361) 2024-12-06T06:15:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,432 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/cce498b65b444ed1a3b7530ae8ec5d9c 2024-12-06T06:15:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/35e08fcad52c4d77a8ad57c8150ae5de is 50, key is test_row_0/B:col10/1733465729306/Put/seqid=0 2024-12-06T06:15:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742246_1422 (size=9657) 2024-12-06T06:15:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,485 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/35e08fcad52c4d77a8ad57c8150ae5de 2024-12-06T06:15:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/156cf8fa3bf84d23a01ea6b3cb422722 is 50, key is test_row_0/C:col10/1733465729306/Put/seqid=0 2024-12-06T06:15:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742247_1423 (size=9657) 2024-12-06T06:15:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,566 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/156cf8fa3bf84d23a01ea6b3cb422722 2024-12-06T06:15:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/cce498b65b444ed1a3b7530ae8ec5d9c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/cce498b65b444ed1a3b7530ae8ec5d9c 2024-12-06T06:15:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,579 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/cce498b65b444ed1a3b7530ae8ec5d9c, entries=100, sequenceid=110, filesize=21.8 K 2024-12-06T06:15:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/35e08fcad52c4d77a8ad57c8150ae5de as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/35e08fcad52c4d77a8ad57c8150ae5de 2024-12-06T06:15:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,583 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/35e08fcad52c4d77a8ad57c8150ae5de, entries=100, sequenceid=110, filesize=9.4 K 2024-12-06T06:15:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/156cf8fa3bf84d23a01ea6b3cb422722 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/156cf8fa3bf84d23a01ea6b3cb422722 2024-12-06T06:15:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,591 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/156cf8fa3bf84d23a01ea6b3cb422722, entries=100, sequenceid=110, filesize=9.4 K 2024-12-06T06:15:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,592 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9d97d8dc224d0a067fc9941f0b9f555b in 590ms, sequenceid=110, compaction requested=true 2024-12-06T06:15:31,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:31,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-06T06:15:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-06T06:15:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-06T06:15:31,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9730 sec 2024-12-06T06:15:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.9810 sec 2024-12-06T06:15:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:15:31,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:31,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:31,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:31,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:31,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:31,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206cc039f9a453840d0b3156571512c2d19_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465731708/Put/seqid=0 2024-12-06T06:15:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742248_1424 (size=17034) 2024-12-06T06:15:31,739 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:31,743 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206cc039f9a453840d0b3156571512c2d19_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cc039f9a453840d0b3156571512c2d19_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:31,744 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/3a6770a62c5b411a94f5e1ca0f6ff819, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:31,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/3a6770a62c5b411a94f5e1ca0f6ff819 is 175, key is test_row_0/A:col10/1733465731708/Put/seqid=0 2024-12-06T06:15:31,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742249_1425 (size=48139) 2024-12-06T06:15:31,749 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/3a6770a62c5b411a94f5e1ca0f6ff819 2024-12-06T06:15:31,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/b77b12337a0e4df6a42dea06220bddf1 is 50, key is test_row_0/B:col10/1733465731708/Put/seqid=0 2024-12-06T06:15:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742250_1426 (size=12001) 2024-12-06T06:15:31,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/b77b12337a0e4df6a42dea06220bddf1 2024-12-06T06:15:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T06:15:31,764 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:31,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-06T06:15:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T06:15:31,767 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:31,767 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:31,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:31,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/0265f5031a8d46a99e5d44762bd8bd19 is 50, key is test_row_0/C:col10/1733465731708/Put/seqid=0 2024-12-06T06:15:31,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742251_1427 (size=12001) 2024-12-06T06:15:31,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465731814, max timestamp=1733465731846Num uniques 859; 2024-12-06T06:15:31,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T06:15:31,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465791866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:31,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:31,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465791886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:31,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-06T06:15:31,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:31,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:31,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:31,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:31,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:32,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465791993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465792014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-06T06:15:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:32,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T06:15:32,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/0265f5031a8d46a99e5d44762bd8bd19 2024-12-06T06:15:32,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/3a6770a62c5b411a94f5e1ca0f6ff819 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/3a6770a62c5b411a94f5e1ca0f6ff819 2024-12-06T06:15:32,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/3a6770a62c5b411a94f5e1ca0f6ff819, entries=250, sequenceid=120, filesize=47.0 K 2024-12-06T06:15:32,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/b77b12337a0e4df6a42dea06220bddf1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b77b12337a0e4df6a42dea06220bddf1 2024-12-06T06:15:32,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b77b12337a0e4df6a42dea06220bddf1, entries=150, sequenceid=120, filesize=11.7 K 2024-12-06T06:15:32,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/0265f5031a8d46a99e5d44762bd8bd19 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0265f5031a8d46a99e5d44762bd8bd19 2024-12-06T06:15:32,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0265f5031a8d46a99e5d44762bd8bd19, entries=150, sequenceid=120, filesize=11.7 K 2024-12-06T06:15:32,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for 9d97d8dc224d0a067fc9941f0b9f555b in 487ms, sequenceid=120, compaction requested=true 2024-12-06T06:15:32,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:32,200 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:32,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:32,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:32,201 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:32,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:32,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:32,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:32,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:32,203 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:32,203 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/B is initiating minor compaction (all files) 2024-12-06T06:15:32,203 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/B in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:32,203 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/084309166d7e4535b61acfba19afd7d0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/8377328f08934d9d9ab93ef09bff1de8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/35e08fcad52c4d77a8ad57c8150ae5de, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b77b12337a0e4df6a42dea06220bddf1] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=44.7 K 2024-12-06T06:15:32,203 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149697 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:32,203 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/A is initiating minor compaction (all files) 2024-12-06T06:15:32,203 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/A in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:32,204 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/1a1160b096034111bdf9da676ee4db6b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/a9791c4bbbc34c1bb131dd7f0c2424b8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/cce498b65b444ed1a3b7530ae8ec5d9c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/3a6770a62c5b411a94f5e1ca0f6ff819] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=146.2 K 2024-12-06T06:15:32,204 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:32,204 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/1a1160b096034111bdf9da676ee4db6b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/a9791c4bbbc34c1bb131dd7f0c2424b8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/cce498b65b444ed1a3b7530ae8ec5d9c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/3a6770a62c5b411a94f5e1ca0f6ff819] 2024-12-06T06:15:32,204 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 084309166d7e4535b61acfba19afd7d0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465726698 2024-12-06T06:15:32,204 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1160b096034111bdf9da676ee4db6b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465726698 2024-12-06T06:15:32,204 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8377328f08934d9d9ab93ef09bff1de8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465728071 2024-12-06T06:15:32,204 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9791c4bbbc34c1bb131dd7f0c2424b8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465728034 2024-12-06T06:15:32,205 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 35e08fcad52c4d77a8ad57c8150ae5de, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733465729295 2024-12-06T06:15:32,205 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b77b12337a0e4df6a42dea06220bddf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733465731650 2024-12-06T06:15:32,205 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cce498b65b444ed1a3b7530ae8ec5d9c, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733465729295 2024-12-06T06:15:32,205 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a6770a62c5b411a94f5e1ca0f6ff819, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733465731599 2024-12-06T06:15:32,213 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:32,218 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:32,219 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/cb0ed3c5604f4877b8bfa46caf32b82e is 50, key is test_row_0/B:col10/1733465731708/Put/seqid=0 2024-12-06T06:15:32,225 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412065017f72357ff4326bba1b60366209433_9d97d8dc224d0a067fc9941f0b9f555b store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:32,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-06T06:15:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:32,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:32,226 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=136.42 KB heapSize=401.56 KB 2024-12-06T06:15:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:32,227 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412065017f72357ff4326bba1b60366209433_9d97d8dc224d0a067fc9941f0b9f555b, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:32,228 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065017f72357ff4326bba1b60366209433_9d97d8dc224d0a067fc9941f0b9f555b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:32,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742252_1428 (size=12241) 2024-12-06T06:15:32,239 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/cb0ed3c5604f4877b8bfa46caf32b82e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/cb0ed3c5604f4877b8bfa46caf32b82e 2024-12-06T06:15:32,247 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/B of 9d97d8dc224d0a067fc9941f0b9f555b into cb0ed3c5604f4877b8bfa46caf32b82e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:32,247 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:32,247 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/B, priority=12, startTime=1733465732201; duration=0sec 2024-12-06T06:15:32,247 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:32,247 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:B 2024-12-06T06:15:32,247 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:32,249 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:32,249 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/C is initiating minor compaction (all files) 2024-12-06T06:15:32,249 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/C in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:32,249 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/3bdcea62e4484b229deb773683bd256a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/029e377109bc473181695d682fff929d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/156cf8fa3bf84d23a01ea6b3cb422722, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0265f5031a8d46a99e5d44762bd8bd19] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=44.7 K 2024-12-06T06:15:32,249 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bdcea62e4484b229deb773683bd256a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733465726698 2024-12-06T06:15:32,251 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 029e377109bc473181695d682fff929d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465728071 2024-12-06T06:15:32,251 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 156cf8fa3bf84d23a01ea6b3cb422722, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733465729295 2024-12-06T06:15:32,251 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0265f5031a8d46a99e5d44762bd8bd19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733465731650 2024-12-06T06:15:32,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742253_1429 (size=4469) 2024-12-06T06:15:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fbdda64d1168450e8d3120ef8dd44309_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465732225/Put/seqid=0 2024-12-06T06:15:32,258 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#A#compaction#368 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:32,258 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/454545e69cd3442e90080009c67c7436 is 175, key is test_row_0/A:col10/1733465731708/Put/seqid=0 2024-12-06T06:15:32,260 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:32,261 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/95d46486d99c45f0b28d6795c9cb2acd is 50, key is test_row_0/C:col10/1733465731708/Put/seqid=0 2024-12-06T06:15:32,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742254_1430 (size=31195) 2024-12-06T06:15:32,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742256_1432 (size=12241) 2024-12-06T06:15:32,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742255_1431 (size=14794) 2024-12-06T06:15:32,270 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/454545e69cd3442e90080009c67c7436 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/454545e69cd3442e90080009c67c7436 2024-12-06T06:15:32,271 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/95d46486d99c45f0b28d6795c9cb2acd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/95d46486d99c45f0b28d6795c9cb2acd 2024-12-06T06:15:32,275 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/A of 9d97d8dc224d0a067fc9941f0b9f555b into 454545e69cd3442e90080009c67c7436(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:32,276 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:32,276 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/A, priority=12, startTime=1733465732200; duration=0sec 2024-12-06T06:15:32,276 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:32,276 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:A 2024-12-06T06:15:32,284 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/C of 9d97d8dc224d0a067fc9941f0b9f555b into 95d46486d99c45f0b28d6795c9cb2acd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:32,284 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:32,284 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/C, priority=12, startTime=1733465732201; duration=0sec 2024-12-06T06:15:32,285 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:32,285 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:C 2024-12-06T06:15:32,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465792277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465792278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T06:15:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465792403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465792403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465792614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465792614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:32,674 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fbdda64d1168450e8d3120ef8dd44309_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fbdda64d1168450e8d3120ef8dd44309_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:32,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59 is 175, key is test_row_0/A:col10/1733465732225/Put/seqid=0 2024-12-06T06:15:32,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742257_1433 (size=39749) 2024-12-06T06:15:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T06:15:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465792934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465792934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:33,081 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59 2024-12-06T06:15:33,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/e4615a0d959a48c5a84d432882e6ae0e is 50, key is test_row_0/B:col10/1733465732225/Put/seqid=0 2024-12-06T06:15:33,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742258_1434 (size=12151) 2024-12-06T06:15:33,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465793454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:33,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465793454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:33,493 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/e4615a0d959a48c5a84d432882e6ae0e 2024-12-06T06:15:33,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/354cdd3e9af14b63a8e58e345c0b075e is 50, key is test_row_0/C:col10/1733465732225/Put/seqid=0 2024-12-06T06:15:33,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742259_1435 (size=12151) 2024-12-06T06:15:33,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465793773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:33,786 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:33,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465793784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:33,794 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:33,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465793794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:33,803 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in 9d97d8dc224d0a067fc9941f0b9f555b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:33,904 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/354cdd3e9af14b63a8e58e345c0b075e 2024-12-06T06:15:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59 2024-12-06T06:15:33,912 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59, entries=200, sequenceid=148, filesize=38.8 K 2024-12-06T06:15:33,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/e4615a0d959a48c5a84d432882e6ae0e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/e4615a0d959a48c5a84d432882e6ae0e 2024-12-06T06:15:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T06:15:33,917 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/e4615a0d959a48c5a84d432882e6ae0e, entries=150, sequenceid=148, filesize=11.9 K 2024-12-06T06:15:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/354cdd3e9af14b63a8e58e345c0b075e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/354cdd3e9af14b63a8e58e345c0b075e 2024-12-06T06:15:33,921 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/354cdd3e9af14b63a8e58e345c0b075e, entries=150, sequenceid=148, filesize=11.9 K 2024-12-06T06:15:33,922 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~401.52 KB/411152, currentSize=40.25 KB/41220 for 9d97d8dc224d0a067fc9941f0b9f555b in 1696ms, sequenceid=148, compaction requested=false 2024-12-06T06:15:33,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:33,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:33,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-06T06:15:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-06T06:15:33,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-06T06:15:33,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1570 sec 2024-12-06T06:15:33,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1620 sec 2024-12-06T06:15:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:34,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:15:34,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:34,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:34,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:34,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:34,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:34,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:34,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206afcd80f58f2b4cf4a30c54d3e37b0520_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465734475/Put/seqid=0 2024-12-06T06:15:34,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742260_1436 (size=14794) 2024-12-06T06:15:34,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465734518, max timestamp=1733465734551Num uniques 859; 2024-12-06T06:15:34,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:15:34,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465734518, max timestamp=1733465734551Num uniques 859; 2024-12-06T06:15:34,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465734518, max timestamp=1733465734551Num uniques 859; 2024-12-06T06:15:34,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:15:34,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:15:34,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465794664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:34,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465794668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:34,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465794774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:34,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465794784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:34,889 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:34,892 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206afcd80f58f2b4cf4a30c54d3e37b0520_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206afcd80f58f2b4cf4a30c54d3e37b0520_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:34,893 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/4a57d14d0430439fa23384e72ae0873f, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:34,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/4a57d14d0430439fa23384e72ae0873f is 175, key is test_row_0/A:col10/1733465734475/Put/seqid=0 2024-12-06T06:15:34,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742261_1437 (size=39749) 2024-12-06T06:15:35,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465794993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:35,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465795004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:35,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/4a57d14d0430439fa23384e72ae0873f 2024-12-06T06:15:35,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/2e8612b5989943ebaa74f5740967f157 is 50, key is test_row_0/B:col10/1733465734475/Put/seqid=0 2024-12-06T06:15:35,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742262_1438 (size=12151) 2024-12-06T06:15:35,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465795314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:35,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465795323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:35,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/2e8612b5989943ebaa74f5740967f157 2024-12-06T06:15:35,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/4c5afdd345f1498abc37d3bf384aac7d is 50, key is test_row_0/C:col10/1733465734475/Put/seqid=0 2024-12-06T06:15:35,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742263_1439 (size=12151) 2024-12-06T06:15:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465795834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:35,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465795843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-06T06:15:35,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:35,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-06T06:15:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T06:15:35,927 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:35,928 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:35,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T06:15:36,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T06:15:36,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:36,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:36,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/4c5afdd345f1498abc37d3bf384aac7d 2024-12-06T06:15:36,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/4a57d14d0430439fa23384e72ae0873f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4a57d14d0430439fa23384e72ae0873f 2024-12-06T06:15:36,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4a57d14d0430439fa23384e72ae0873f, entries=200, sequenceid=161, filesize=38.8 K 2024-12-06T06:15:36,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/2e8612b5989943ebaa74f5740967f157 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/2e8612b5989943ebaa74f5740967f157 2024-12-06T06:15:36,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/2e8612b5989943ebaa74f5740967f157, entries=150, sequenceid=161, filesize=11.9 K 2024-12-06T06:15:36,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/4c5afdd345f1498abc37d3bf384aac7d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/4c5afdd345f1498abc37d3bf384aac7d 2024-12-06T06:15:36,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/4c5afdd345f1498abc37d3bf384aac7d, entries=150, sequenceid=161, filesize=11.9 K 2024-12-06T06:15:36,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9d97d8dc224d0a067fc9941f0b9f555b in 1658ms, sequenceid=161, compaction requested=true 2024-12-06T06:15:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:36,135 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:36,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:36,135 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:36,136 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:36,136 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:36,136 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/A is initiating minor compaction (all files) 2024-12-06T06:15:36,136 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/B is initiating minor compaction (all files) 2024-12-06T06:15:36,136 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/A in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:36,136 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/B in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:36,136 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/454545e69cd3442e90080009c67c7436, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4a57d14d0430439fa23384e72ae0873f] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=108.1 K 2024-12-06T06:15:36,136 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:36,136 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/454545e69cd3442e90080009c67c7436, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4a57d14d0430439fa23384e72ae0873f] 2024-12-06T06:15:36,136 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/cb0ed3c5604f4877b8bfa46caf32b82e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/e4615a0d959a48c5a84d432882e6ae0e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/2e8612b5989943ebaa74f5740967f157] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=35.7 K 2024-12-06T06:15:36,137 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454545e69cd3442e90080009c67c7436, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733465731650 2024-12-06T06:15:36,137 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0ed3c5604f4877b8bfa46caf32b82e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733465731650 2024-12-06T06:15:36,137 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a7f7ed1f72d4bec90db3b0f9ca3bb59, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733465731862 2024-12-06T06:15:36,137 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e4615a0d959a48c5a84d432882e6ae0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733465731862 2024-12-06T06:15:36,137 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a57d14d0430439fa23384e72ae0873f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733465732262 2024-12-06T06:15:36,137 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e8612b5989943ebaa74f5740967f157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733465732262 2024-12-06T06:15:36,143 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:36,147 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:36,147 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/5c40a2900e894dc29bf49dcdb4be062a is 50, key is test_row_0/B:col10/1733465734475/Put/seqid=0 2024-12-06T06:15:36,148 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120613abdb786d344644afcdbdd92a3ec1b9_9d97d8dc224d0a067fc9941f0b9f555b store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:36,150 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120613abdb786d344644afcdbdd92a3ec1b9_9d97d8dc224d0a067fc9941f0b9f555b, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:36,151 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120613abdb786d344644afcdbdd92a3ec1b9_9d97d8dc224d0a067fc9941f0b9f555b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:36,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742264_1440 (size=12493) 2024-12-06T06:15:36,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742265_1441 (size=4469) 2024-12-06T06:15:36,162 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#A#compaction#377 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:36,162 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/bb5e912af0754f4abb37b46d604e5800 is 175, key is test_row_0/A:col10/1733465734475/Put/seqid=0 2024-12-06T06:15:36,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742266_1442 (size=31447) 2024-12-06T06:15:36,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T06:15:36,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:36,233 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-06T06:15:36,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:36,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:36,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:36,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:36,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:36,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:36,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a0d9e851755a478c9da04ce76a50fd83_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465734635/Put/seqid=0 2024-12-06T06:15:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T06:15:36,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742267_1443 (size=12304) 2024-12-06T06:15:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T06:15:36,564 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/5c40a2900e894dc29bf49dcdb4be062a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/5c40a2900e894dc29bf49dcdb4be062a 2024-12-06T06:15:36,568 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/B of 9d97d8dc224d0a067fc9941f0b9f555b into 5c40a2900e894dc29bf49dcdb4be062a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:36,568 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:36,568 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/B, priority=13, startTime=1733465736135; duration=0sec 2024-12-06T06:15:36,568 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:36,569 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:B 2024-12-06T06:15:36,569 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:36,570 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:36,570 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/C is initiating minor compaction (all files) 2024-12-06T06:15:36,570 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/C in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:36,570 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/95d46486d99c45f0b28d6795c9cb2acd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/354cdd3e9af14b63a8e58e345c0b075e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/4c5afdd345f1498abc37d3bf384aac7d] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=35.7 K 2024-12-06T06:15:36,571 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d46486d99c45f0b28d6795c9cb2acd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733465731650 2024-12-06T06:15:36,571 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 354cdd3e9af14b63a8e58e345c0b075e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733465731862 2024-12-06T06:15:36,571 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c5afdd345f1498abc37d3bf384aac7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733465732262 2024-12-06T06:15:36,572 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/bb5e912af0754f4abb37b46d604e5800 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/bb5e912af0754f4abb37b46d604e5800 2024-12-06T06:15:36,576 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/A of 9d97d8dc224d0a067fc9941f0b9f555b into bb5e912af0754f4abb37b46d604e5800(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:36,576 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:36,576 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/A, priority=13, startTime=1733465736135; duration=0sec 2024-12-06T06:15:36,576 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:36,576 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:A 2024-12-06T06:15:36,578 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:36,579 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/c97fbef91fb44cf984ec9c60089d53db is 50, key is test_row_0/C:col10/1733465734475/Put/seqid=0 2024-12-06T06:15:36,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742268_1444 (size=12493) 2024-12-06T06:15:36,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:36,649 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a0d9e851755a478c9da04ce76a50fd83_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a0d9e851755a478c9da04ce76a50fd83_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:36,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/4fa032b14626480b935b7312191a9a0c, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:36,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/4fa032b14626480b935b7312191a9a0c is 175, key is test_row_0/A:col10/1733465734635/Put/seqid=0 2024-12-06T06:15:36,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742269_1445 (size=31105) 2024-12-06T06:15:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:36,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:36,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465796910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:36,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:36,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:36,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465796911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:36,987 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/c97fbef91fb44cf984ec9c60089d53db as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/c97fbef91fb44cf984ec9c60089d53db 2024-12-06T06:15:36,991 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/C of 9d97d8dc224d0a067fc9941f0b9f555b into c97fbef91fb44cf984ec9c60089d53db(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:36,991 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:36,991 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/C, priority=13, startTime=1733465736135; duration=0sec 2024-12-06T06:15:36,991 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:36,991 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:C 2024-12-06T06:15:37,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465797014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:37,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465797034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:37,054 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/4fa032b14626480b935b7312191a9a0c 2024-12-06T06:15:37,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/a9644c9d680145e0b056d43f3130f5b9 is 50, key is test_row_0/B:col10/1733465734635/Put/seqid=0 2024-12-06T06:15:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T06:15:37,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742270_1446 (size=12151) 2024-12-06T06:15:37,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465797234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:37,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465797244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:37,470 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/a9644c9d680145e0b056d43f3130f5b9 2024-12-06T06:15:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8daa628fabbb43749ea6793d7fbd268d is 50, key is test_row_0/C:col10/1733465734635/Put/seqid=0 2024-12-06T06:15:37,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742271_1447 (size=12151) 2024-12-06T06:15:37,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465797543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:37,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465797564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:37,883 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8daa628fabbb43749ea6793d7fbd268d 2024-12-06T06:15:37,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/4fa032b14626480b935b7312191a9a0c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4fa032b14626480b935b7312191a9a0c 2024-12-06T06:15:37,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4fa032b14626480b935b7312191a9a0c, entries=150, sequenceid=197, filesize=30.4 K 2024-12-06T06:15:37,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/a9644c9d680145e0b056d43f3130f5b9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/a9644c9d680145e0b056d43f3130f5b9 2024-12-06T06:15:37,895 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/a9644c9d680145e0b056d43f3130f5b9, entries=150, sequenceid=197, filesize=11.9 K 2024-12-06T06:15:37,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8daa628fabbb43749ea6793d7fbd268d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8daa628fabbb43749ea6793d7fbd268d 2024-12-06T06:15:37,898 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8daa628fabbb43749ea6793d7fbd268d, entries=150, sequenceid=197, filesize=11.9 K 2024-12-06T06:15:37,899 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 9d97d8dc224d0a067fc9941f0b9f555b in 1666ms, sequenceid=197, compaction requested=false 2024-12-06T06:15:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-06T06:15:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-06T06:15:37,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-06T06:15:37,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9730 sec 2024-12-06T06:15:37,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.9780 sec 2024-12-06T06:15:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:38,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T06:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:38,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a1f4bf98794548c3b3da7b7a4880f657_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465736876/Put/seqid=0 2024-12-06T06:15:38,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742272_1448 (size=14794) 2024-12-06T06:15:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T06:15:38,084 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:38,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-06T06:15:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T06:15:38,087 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:38,087 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:38,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:38,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465798160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465738118, max timestamp=1733465738160Num uniques 859; 2024-12-06T06:15:38,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:15:38,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465738118, max timestamp=1733465738160Num uniques 859; 2024-12-06T06:15:38,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:15:38,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733465738118, max timestamp=1733465738160Num uniques 859; 2024-12-06T06:15:38,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-06T06:15:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T06:15:38,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465798290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465798298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:38,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T06:15:38,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465798414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:38,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465798424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:38,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:38,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a1f4bf98794548c3b3da7b7a4880f657_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a1f4bf98794548c3b3da7b7a4880f657_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:38,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/17f0eb50c0e04c0aa43e9973ad36143b, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:38,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/17f0eb50c0e04c0aa43e9973ad36143b is 175, key is test_row_0/A:col10/1733465736876/Put/seqid=0 2024-12-06T06:15:38,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742273_1449 (size=39749) 2024-12-06T06:15:38,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465798633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465798644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:38,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T06:15:38,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:38,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:38,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/17f0eb50c0e04c0aa43e9973ad36143b 2024-12-06T06:15:38,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/74202903c2244f4381cdf6b03fd6e9e3 is 50, key is test_row_0/B:col10/1733465736876/Put/seqid=0 2024-12-06T06:15:38,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742274_1450 (size=12151) 2024-12-06T06:15:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465798954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465798954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:39,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T06:15:39,290 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T06:15:39,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/74202903c2244f4381cdf6b03fd6e9e3 2024-12-06T06:15:39,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/cb065e658d104b169dd43e986227d629 is 50, key is test_row_0/C:col10/1733465736876/Put/seqid=0 2024-12-06T06:15:39,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:39,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:39,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:39,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742275_1451 (size=12151) 2024-12-06T06:15:39,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/cb065e658d104b169dd43e986227d629 2024-12-06T06:15:39,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/17f0eb50c0e04c0aa43e9973ad36143b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/17f0eb50c0e04c0aa43e9973ad36143b 2024-12-06T06:15:39,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/17f0eb50c0e04c0aa43e9973ad36143b, entries=200, sequenceid=211, filesize=38.8 K 2024-12-06T06:15:39,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/74202903c2244f4381cdf6b03fd6e9e3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/74202903c2244f4381cdf6b03fd6e9e3 2024-12-06T06:15:39,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/74202903c2244f4381cdf6b03fd6e9e3, entries=150, sequenceid=211, filesize=11.9 K 2024-12-06T06:15:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/cb065e658d104b169dd43e986227d629 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/cb065e658d104b169dd43e986227d629 2024-12-06T06:15:39,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/cb065e658d104b169dd43e986227d629, entries=150, sequenceid=211, filesize=11.9 K 2024-12-06T06:15:39,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9d97d8dc224d0a067fc9941f0b9f555b in 1317ms, sequenceid=211, compaction requested=true 2024-12-06T06:15:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:39,375 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:39,375 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:39,380 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:39,380 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:39,381 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/B is initiating minor compaction (all files) 2024-12-06T06:15:39,381 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/A is initiating minor compaction (all files) 2024-12-06T06:15:39,381 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/B in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,381 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/A in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,381 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/5c40a2900e894dc29bf49dcdb4be062a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/a9644c9d680145e0b056d43f3130f5b9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/74202903c2244f4381cdf6b03fd6e9e3] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=35.9 K 2024-12-06T06:15:39,381 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/bb5e912af0754f4abb37b46d604e5800, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4fa032b14626480b935b7312191a9a0c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/17f0eb50c0e04c0aa43e9973ad36143b] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=99.9 K 2024-12-06T06:15:39,381 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,381 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/bb5e912af0754f4abb37b46d604e5800, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4fa032b14626480b935b7312191a9a0c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/17f0eb50c0e04c0aa43e9973ad36143b] 2024-12-06T06:15:39,381 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c40a2900e894dc29bf49dcdb4be062a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733465732262 2024-12-06T06:15:39,381 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a9644c9d680145e0b056d43f3130f5b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733465734635 2024-12-06T06:15:39,381 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5e912af0754f4abb37b46d604e5800, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733465732262 2024-12-06T06:15:39,382 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 74202903c2244f4381cdf6b03fd6e9e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733465736876 2024-12-06T06:15:39,382 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa032b14626480b935b7312191a9a0c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733465734635 2024-12-06T06:15:39,382 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f0eb50c0e04c0aa43e9973ad36143b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733465736876 2024-12-06T06:15:39,389 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#B#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:39,390 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/ac743e27d51e48ddbdfa09f61e163550 is 50, key is test_row_0/B:col10/1733465736876/Put/seqid=0 2024-12-06T06:15:39,393 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:39,398 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120665168d6dd8e54313aea5a6816130426a_9d97d8dc224d0a067fc9941f0b9f555b store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:39,400 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120665168d6dd8e54313aea5a6816130426a_9d97d8dc224d0a067fc9941f0b9f555b, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:39,400 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120665168d6dd8e54313aea5a6816130426a_9d97d8dc224d0a067fc9941f0b9f555b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:39,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742276_1452 (size=12595) 2024-12-06T06:15:39,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742277_1453 (size=4469) 2024-12-06T06:15:39,452 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#A#compaction#387 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:39,452 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/d213a87eff9a49dc8aa313d3ba3de37d is 175, key is test_row_0/A:col10/1733465736876/Put/seqid=0 2024-12-06T06:15:39,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742278_1454 (size=31549) 2024-12-06T06:15:39,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T06:15:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,465 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-06T06:15:39,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:39,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:39,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:39,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:39,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:39,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:15:39,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065c48cbb11a284dfdb6d5294d03fff4b5_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465738221/Put/seqid=0 2024-12-06T06:15:39,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742279_1455 (size=12304) 2024-12-06T06:15:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:39,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:39,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465799526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:39,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465799528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:39,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465799654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:39,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465799654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:39,817 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/ac743e27d51e48ddbdfa09f61e163550 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/ac743e27d51e48ddbdfa09f61e163550 2024-12-06T06:15:39,825 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/B of 9d97d8dc224d0a067fc9941f0b9f555b into ac743e27d51e48ddbdfa09f61e163550(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:39,825 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:39,825 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/B, priority=13, startTime=1733465739375; duration=0sec 2024-12-06T06:15:39,825 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:39,825 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:B 2024-12-06T06:15:39,825 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:39,826 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:39,826 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/C is initiating minor compaction (all files) 2024-12-06T06:15:39,826 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/C in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:39,826 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/c97fbef91fb44cf984ec9c60089d53db, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8daa628fabbb43749ea6793d7fbd268d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/cb065e658d104b169dd43e986227d629] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=35.9 K 2024-12-06T06:15:39,827 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting c97fbef91fb44cf984ec9c60089d53db, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733465732262 2024-12-06T06:15:39,827 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8daa628fabbb43749ea6793d7fbd268d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733465734635 2024-12-06T06:15:39,827 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting cb065e658d104b169dd43e986227d629, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733465736876 2024-12-06T06:15:39,855 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#C#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:39,855 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/457dae492e994fcd8100d7f65039d753 is 50, key is test_row_0/C:col10/1733465736876/Put/seqid=0 2024-12-06T06:15:39,865 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/d213a87eff9a49dc8aa313d3ba3de37d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/d213a87eff9a49dc8aa313d3ba3de37d 2024-12-06T06:15:39,872 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/A of 9d97d8dc224d0a067fc9941f0b9f555b into d213a87eff9a49dc8aa313d3ba3de37d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:39,872 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:39,872 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/A, priority=13, startTime=1733465739375; duration=0sec 2024-12-06T06:15:39,872 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:39,872 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:A 2024-12-06T06:15:39,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:39,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465799874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465799874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:39,887 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065c48cbb11a284dfdb6d5294d03fff4b5_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065c48cbb11a284dfdb6d5294d03fff4b5_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/24e65b91569e4df6a74363e1aa1e2abe, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/24e65b91569e4df6a74363e1aa1e2abe is 175, key is test_row_0/A:col10/1733465738221/Put/seqid=0 2024-12-06T06:15:39,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742280_1456 (size=12595) 2024-12-06T06:15:39,908 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/457dae492e994fcd8100d7f65039d753 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/457dae492e994fcd8100d7f65039d753 2024-12-06T06:15:39,913 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/C of 9d97d8dc224d0a067fc9941f0b9f555b into 457dae492e994fcd8100d7f65039d753(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:39,913 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:39,913 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/C, priority=13, startTime=1733465739375; duration=0sec 2024-12-06T06:15:39,913 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:39,913 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:C 2024-12-06T06:15:39,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742281_1457 (size=31105) 2024-12-06T06:15:40,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465800195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:40,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465800195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T06:15:40,338 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/24e65b91569e4df6a74363e1aa1e2abe 2024-12-06T06:15:40,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/071b5eda0c2e41f7a2297f0ee7c34327 is 50, key is test_row_0/B:col10/1733465738221/Put/seqid=0 2024-12-06T06:15:40,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742282_1458 (size=12151) 2024-12-06T06:15:40,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465800704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:40,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465800704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:40,811 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/071b5eda0c2e41f7a2297f0ee7c34327 2024-12-06T06:15:40,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8e92803818dd40029e5eec278dde3f59 is 50, key is test_row_0/C:col10/1733465738221/Put/seqid=0 2024-12-06T06:15:40,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742283_1459 (size=12151) 2024-12-06T06:15:41,225 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8e92803818dd40029e5eec278dde3f59 2024-12-06T06:15:41,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/24e65b91569e4df6a74363e1aa1e2abe as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/24e65b91569e4df6a74363e1aa1e2abe 2024-12-06T06:15:41,237 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/24e65b91569e4df6a74363e1aa1e2abe, entries=150, sequenceid=245, filesize=30.4 K 2024-12-06T06:15:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/071b5eda0c2e41f7a2297f0ee7c34327 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/071b5eda0c2e41f7a2297f0ee7c34327 2024-12-06T06:15:41,242 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/071b5eda0c2e41f7a2297f0ee7c34327, entries=150, sequenceid=245, filesize=11.9 K 2024-12-06T06:15:41,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8e92803818dd40029e5eec278dde3f59 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e92803818dd40029e5eec278dde3f59 2024-12-06T06:15:41,247 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e92803818dd40029e5eec278dde3f59, entries=150, sequenceid=245, filesize=11.9 K 2024-12-06T06:15:41,248 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 9d97d8dc224d0a067fc9941f0b9f555b in 1783ms, sequenceid=245, compaction requested=false 2024-12-06T06:15:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-06T06:15:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-06T06:15:41,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-06T06:15:41,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1700 sec 2024-12-06T06:15:41,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.1760 sec 2024-12-06T06:15:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:41,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T06:15:41,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:41,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:41,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:41,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:41,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:41,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:41,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a96f8b28b7974709a40cc8b6e1a076b7_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465741732/Put/seqid=0 2024-12-06T06:15:41,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742284_1460 (size=14844) 2024-12-06T06:15:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465801860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:41,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465801862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:41,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465801974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:41,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465801983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:41,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:41,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:41,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:42,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465802194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:42,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465802194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:42,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:42,224 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a96f8b28b7974709a40cc8b6e1a076b7_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a96f8b28b7974709a40cc8b6e1a076b7_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:42,225 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/0d6d0288c9594a328d6071652e313410, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:42,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/0d6d0288c9594a328d6071652e313410 is 175, key is test_row_0/A:col10/1733465741732/Put/seqid=0 2024-12-06T06:15:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T06:15:42,254 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:42,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-06T06:15:42,256 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:42,257 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:42,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:42,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T06:15:42,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742285_1461 (size=39799) 2024-12-06T06:15:42,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/0d6d0288c9594a328d6071652e313410 2024-12-06T06:15:42,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/bcf8118fcaec48e185c95e82bf246f35 is 50, key is test_row_0/B:col10/1733465741732/Put/seqid=0 2024-12-06T06:15:42,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742286_1462 (size=12201) 2024-12-06T06:15:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T06:15:42,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T06:15:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:42,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465802513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:42,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465802513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:42,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T06:15:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:42,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:42,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T06:15:42,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/bcf8118fcaec48e185c95e82bf246f35 2024-12-06T06:15:42,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/ea61a1aecc5b49e0babb3c540f89c0aa is 50, key is test_row_0/C:col10/1733465741732/Put/seqid=0 2024-12-06T06:15:42,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742287_1463 (size=12201) 2024-12-06T06:15:42,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T06:15:42,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:42,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:42,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:42,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T06:15:42,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:42,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:42,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:42,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T06:15:43,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T06:15:43,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:43,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:43,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:43,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:43,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:43,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465803034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:43,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:43,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465803034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:43,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/ea61a1aecc5b49e0babb3c540f89c0aa 2024-12-06T06:15:43,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/0d6d0288c9594a328d6071652e313410 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/0d6d0288c9594a328d6071652e313410 2024-12-06T06:15:43,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/0d6d0288c9594a328d6071652e313410, entries=200, sequenceid=260, filesize=38.9 K 2024-12-06T06:15:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/bcf8118fcaec48e185c95e82bf246f35 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/bcf8118fcaec48e185c95e82bf246f35 2024-12-06T06:15:43,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/bcf8118fcaec48e185c95e82bf246f35, entries=150, sequenceid=260, filesize=11.9 K 2024-12-06T06:15:43,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/ea61a1aecc5b49e0babb3c540f89c0aa as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/ea61a1aecc5b49e0babb3c540f89c0aa 2024-12-06T06:15:43,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/ea61a1aecc5b49e0babb3c540f89c0aa, entries=150, sequenceid=260, filesize=11.9 K 2024-12-06T06:15:43,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9d97d8dc224d0a067fc9941f0b9f555b in 1380ms, sequenceid=260, compaction requested=true 2024-12-06T06:15:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:43,115 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:43,115 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:43,116 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:43,116 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:43,116 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/B is initiating minor compaction (all files) 2024-12-06T06:15:43,116 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/A is initiating minor compaction (all files) 2024-12-06T06:15:43,116 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/B in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:43,116 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/A in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:43,116 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/ac743e27d51e48ddbdfa09f61e163550, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/071b5eda0c2e41f7a2297f0ee7c34327, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/bcf8118fcaec48e185c95e82bf246f35] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=36.1 K 2024-12-06T06:15:43,116 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/d213a87eff9a49dc8aa313d3ba3de37d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/24e65b91569e4df6a74363e1aa1e2abe, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/0d6d0288c9594a328d6071652e313410] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=100.1 K 2024-12-06T06:15:43,116 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:43,116 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/d213a87eff9a49dc8aa313d3ba3de37d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/24e65b91569e4df6a74363e1aa1e2abe, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/0d6d0288c9594a328d6071652e313410] 2024-12-06T06:15:43,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ac743e27d51e48ddbdfa09f61e163550, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733465736876 2024-12-06T06:15:43,117 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d213a87eff9a49dc8aa313d3ba3de37d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733465736876 2024-12-06T06:15:43,117 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e65b91569e4df6a74363e1aa1e2abe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733465738221 2024-12-06T06:15:43,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 071b5eda0c2e41f7a2297f0ee7c34327, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733465738221 2024-12-06T06:15:43,117 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d6d0288c9594a328d6071652e313410, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733465739491 2024-12-06T06:15:43,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf8118fcaec48e185c95e82bf246f35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733465739525 2024-12-06T06:15:43,122 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:43,123 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#B#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:43,124 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/7dd83579828344a5a4833715952c3887 is 50, key is test_row_0/B:col10/1733465741732/Put/seqid=0 2024-12-06T06:15:43,126 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120699785fbf2ac644428903b868d2fadce7_9d97d8dc224d0a067fc9941f0b9f555b store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:43,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742288_1464 (size=12747) 2024-12-06T06:15:43,128 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120699785fbf2ac644428903b868d2fadce7_9d97d8dc224d0a067fc9941f0b9f555b, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:43,128 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120699785fbf2ac644428903b868d2fadce7_9d97d8dc224d0a067fc9941f0b9f555b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:43,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742289_1465 (size=4469) 2024-12-06T06:15:43,134 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#A#compaction#396 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:43,135 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/6b20e04c51224083b859f3c9a9b4ef01 is 175, key is test_row_0/A:col10/1733465741732/Put/seqid=0 2024-12-06T06:15:43,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742290_1466 (size=31701) 2024-12-06T06:15:43,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T06:15:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:43,184 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:15:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ca3014f2212f415b9d48b5d2407625a7_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465741858/Put/seqid=0 2024-12-06T06:15:43,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742291_1467 (size=12454) 2024-12-06T06:15:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:43,201 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ca3014f2212f415b9d48b5d2407625a7_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ca3014f2212f415b9d48b5d2407625a7_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/c3919d3e826f44f1862eea1b72115200, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/c3919d3e826f44f1862eea1b72115200 is 175, key is test_row_0/A:col10/1733465741858/Put/seqid=0 2024-12-06T06:15:43,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742292_1468 (size=31255) 2024-12-06T06:15:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T06:15:43,532 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/7dd83579828344a5a4833715952c3887 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/7dd83579828344a5a4833715952c3887 2024-12-06T06:15:43,536 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/B of 9d97d8dc224d0a067fc9941f0b9f555b into 7dd83579828344a5a4833715952c3887(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:43,536 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:43,536 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/B, priority=13, startTime=1733465743115; duration=0sec 2024-12-06T06:15:43,536 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:43,536 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:B 2024-12-06T06:15:43,536 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:43,537 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:43,537 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/C is initiating minor compaction (all files) 2024-12-06T06:15:43,537 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/C in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:43,537 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/457dae492e994fcd8100d7f65039d753, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e92803818dd40029e5eec278dde3f59, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/ea61a1aecc5b49e0babb3c540f89c0aa] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=36.1 K 2024-12-06T06:15:43,537 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 457dae492e994fcd8100d7f65039d753, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733465736876 2024-12-06T06:15:43,538 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e92803818dd40029e5eec278dde3f59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733465738221 2024-12-06T06:15:43,538 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ea61a1aecc5b49e0babb3c540f89c0aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733465739525 2024-12-06T06:15:43,542 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/6b20e04c51224083b859f3c9a9b4ef01 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/6b20e04c51224083b859f3c9a9b4ef01 2024-12-06T06:15:43,544 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:43,545 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8a3658362ada43e6b3b6f0793e489b7c is 50, key is test_row_0/C:col10/1733465741732/Put/seqid=0 2024-12-06T06:15:43,547 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/A of 9d97d8dc224d0a067fc9941f0b9f555b into 6b20e04c51224083b859f3c9a9b4ef01(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:43,547 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:43,547 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/A, priority=13, startTime=1733465743115; duration=0sec 2024-12-06T06:15:43,547 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:43,547 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:A 2024-12-06T06:15:43,549 WARN [regionserver/6336e8474e1c:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-12-06T06:15:43,550 INFO [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2301): Can't archive compacted file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/d213a87eff9a49dc8aa313d3ba3de37d because of either isCompactedAway=true or file has reference, isReferencedInReads=true, refCount=2, skipping for now. 2024-12-06T06:15:43,550 INFO [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2301): Can't archive compacted file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/24e65b91569e4df6a74363e1aa1e2abe because of either isCompactedAway=true or file has reference, isReferencedInReads=true, refCount=2, skipping for now. 2024-12-06T06:15:43,550 INFO [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2301): Can't archive compacted file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/0d6d0288c9594a328d6071652e313410 because of either isCompactedAway=true or file has reference, isReferencedInReads=true, refCount=2, skipping for now. 2024-12-06T06:15:43,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/5188018fb5a04e13af7bdffea1636953, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/260cdeb519dc4edca9cdcef66e1f2277, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/fde34abcd7a4426386b504005e0901e7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/1a1160b096034111bdf9da676ee4db6b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/a9791c4bbbc34c1bb131dd7f0c2424b8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/cce498b65b444ed1a3b7530ae8ec5d9c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/3a6770a62c5b411a94f5e1ca0f6ff819, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/454545e69cd3442e90080009c67c7436, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4a57d14d0430439fa23384e72ae0873f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/bb5e912af0754f4abb37b46d604e5800, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4fa032b14626480b935b7312191a9a0c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/17f0eb50c0e04c0aa43e9973ad36143b] to archive 2024-12-06T06:15:43,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:43,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/5188018fb5a04e13af7bdffea1636953 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/5188018fb5a04e13af7bdffea1636953 2024-12-06T06:15:43,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/260cdeb519dc4edca9cdcef66e1f2277 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/260cdeb519dc4edca9cdcef66e1f2277 2024-12-06T06:15:43,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/fde34abcd7a4426386b504005e0901e7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/fde34abcd7a4426386b504005e0901e7 2024-12-06T06:15:43,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/1a1160b096034111bdf9da676ee4db6b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/1a1160b096034111bdf9da676ee4db6b 2024-12-06T06:15:43,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/a9791c4bbbc34c1bb131dd7f0c2424b8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/a9791c4bbbc34c1bb131dd7f0c2424b8 2024-12-06T06:15:43,560 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/cce498b65b444ed1a3b7530ae8ec5d9c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/cce498b65b444ed1a3b7530ae8ec5d9c 2024-12-06T06:15:43,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/3a6770a62c5b411a94f5e1ca0f6ff819 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/3a6770a62c5b411a94f5e1ca0f6ff819 2024-12-06T06:15:43,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742293_1469 (size=12747) 2024-12-06T06:15:43,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/454545e69cd3442e90080009c67c7436 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/454545e69cd3442e90080009c67c7436 2024-12-06T06:15:43,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2a7f7ed1f72d4bec90db3b0f9ca3bb59 2024-12-06T06:15:43,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4a57d14d0430439fa23384e72ae0873f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4a57d14d0430439fa23384e72ae0873f 2024-12-06T06:15:43,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/bb5e912af0754f4abb37b46d604e5800 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/bb5e912af0754f4abb37b46d604e5800 2024-12-06T06:15:43,575 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/8a3658362ada43e6b3b6f0793e489b7c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8a3658362ada43e6b3b6f0793e489b7c 2024-12-06T06:15:43,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4fa032b14626480b935b7312191a9a0c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/4fa032b14626480b935b7312191a9a0c 2024-12-06T06:15:43,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/17f0eb50c0e04c0aa43e9973ad36143b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/17f0eb50c0e04c0aa43e9973ad36143b 2024-12-06T06:15:43,579 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/C of 9d97d8dc224d0a067fc9941f0b9f555b into 8a3658362ada43e6b3b6f0793e489b7c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:43,579 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:43,579 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/C, priority=13, startTime=1733465743115; duration=0sec 2024-12-06T06:15:43,579 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:43,579 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:C 2024-12-06T06:15:43,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/924ea17898624e3a801a22c759b25974, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/08dd7d9a00824d6490ecd5391e121f38, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/084309166d7e4535b61acfba19afd7d0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/0a6837b5b6b44f3288ba002909e2bd9f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/8377328f08934d9d9ab93ef09bff1de8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/35e08fcad52c4d77a8ad57c8150ae5de, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/cb0ed3c5604f4877b8bfa46caf32b82e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b77b12337a0e4df6a42dea06220bddf1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/e4615a0d959a48c5a84d432882e6ae0e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/5c40a2900e894dc29bf49dcdb4be062a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/2e8612b5989943ebaa74f5740967f157, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/a9644c9d680145e0b056d43f3130f5b9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/ac743e27d51e48ddbdfa09f61e163550, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/74202903c2244f4381cdf6b03fd6e9e3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/071b5eda0c2e41f7a2297f0ee7c34327, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/bcf8118fcaec48e185c95e82bf246f35] to archive 2024-12-06T06:15:43,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:43,582 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/924ea17898624e3a801a22c759b25974 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/924ea17898624e3a801a22c759b25974 2024-12-06T06:15:43,583 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/08dd7d9a00824d6490ecd5391e121f38 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/08dd7d9a00824d6490ecd5391e121f38 2024-12-06T06:15:43,584 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/084309166d7e4535b61acfba19afd7d0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/084309166d7e4535b61acfba19afd7d0 2024-12-06T06:15:43,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/0a6837b5b6b44f3288ba002909e2bd9f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/0a6837b5b6b44f3288ba002909e2bd9f 2024-12-06T06:15:43,586 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/8377328f08934d9d9ab93ef09bff1de8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/8377328f08934d9d9ab93ef09bff1de8 2024-12-06T06:15:43,586 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/35e08fcad52c4d77a8ad57c8150ae5de to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/35e08fcad52c4d77a8ad57c8150ae5de 2024-12-06T06:15:43,587 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/cb0ed3c5604f4877b8bfa46caf32b82e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/cb0ed3c5604f4877b8bfa46caf32b82e 2024-12-06T06:15:43,588 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b77b12337a0e4df6a42dea06220bddf1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b77b12337a0e4df6a42dea06220bddf1 2024-12-06T06:15:43,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/e4615a0d959a48c5a84d432882e6ae0e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/e4615a0d959a48c5a84d432882e6ae0e 2024-12-06T06:15:43,590 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/5c40a2900e894dc29bf49dcdb4be062a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/5c40a2900e894dc29bf49dcdb4be062a 2024-12-06T06:15:43,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/2e8612b5989943ebaa74f5740967f157 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/2e8612b5989943ebaa74f5740967f157 2024-12-06T06:15:43,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/a9644c9d680145e0b056d43f3130f5b9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/a9644c9d680145e0b056d43f3130f5b9 2024-12-06T06:15:43,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/ac743e27d51e48ddbdfa09f61e163550 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/ac743e27d51e48ddbdfa09f61e163550 2024-12-06T06:15:43,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/74202903c2244f4381cdf6b03fd6e9e3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/74202903c2244f4381cdf6b03fd6e9e3 2024-12-06T06:15:43,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/071b5eda0c2e41f7a2297f0ee7c34327 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/071b5eda0c2e41f7a2297f0ee7c34327 2024-12-06T06:15:43,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/bcf8118fcaec48e185c95e82bf246f35 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/bcf8118fcaec48e185c95e82bf246f35 2024-12-06T06:15:43,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/a4e6822156534a33a9ad69cdb31cf103, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/25a31716541c42e6b60c82599ca0ace8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/3bdcea62e4484b229deb773683bd256a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e5b0d03b86c42608e207e70a7c6ce6d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/029e377109bc473181695d682fff929d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/156cf8fa3bf84d23a01ea6b3cb422722, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/95d46486d99c45f0b28d6795c9cb2acd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0265f5031a8d46a99e5d44762bd8bd19, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/354cdd3e9af14b63a8e58e345c0b075e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/c97fbef91fb44cf984ec9c60089d53db, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/4c5afdd345f1498abc37d3bf384aac7d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8daa628fabbb43749ea6793d7fbd268d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/457dae492e994fcd8100d7f65039d753, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/cb065e658d104b169dd43e986227d629, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e92803818dd40029e5eec278dde3f59, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/ea61a1aecc5b49e0babb3c540f89c0aa] to archive 2024-12-06T06:15:43,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:43,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/a4e6822156534a33a9ad69cdb31cf103 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/a4e6822156534a33a9ad69cdb31cf103 2024-12-06T06:15:43,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/25a31716541c42e6b60c82599ca0ace8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/25a31716541c42e6b60c82599ca0ace8 2024-12-06T06:15:43,604 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/3bdcea62e4484b229deb773683bd256a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/3bdcea62e4484b229deb773683bd256a 2024-12-06T06:15:43,605 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e5b0d03b86c42608e207e70a7c6ce6d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e5b0d03b86c42608e207e70a7c6ce6d 2024-12-06T06:15:43,606 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/029e377109bc473181695d682fff929d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/029e377109bc473181695d682fff929d 2024-12-06T06:15:43,607 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/156cf8fa3bf84d23a01ea6b3cb422722 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/156cf8fa3bf84d23a01ea6b3cb422722 2024-12-06T06:15:43,608 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/95d46486d99c45f0b28d6795c9cb2acd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/95d46486d99c45f0b28d6795c9cb2acd 2024-12-06T06:15:43,609 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/c3919d3e826f44f1862eea1b72115200 2024-12-06T06:15:43,610 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0265f5031a8d46a99e5d44762bd8bd19 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0265f5031a8d46a99e5d44762bd8bd19 2024-12-06T06:15:43,611 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/354cdd3e9af14b63a8e58e345c0b075e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/354cdd3e9af14b63a8e58e345c0b075e 2024-12-06T06:15:43,613 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/c97fbef91fb44cf984ec9c60089d53db to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/c97fbef91fb44cf984ec9c60089d53db 2024-12-06T06:15:43,614 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/4c5afdd345f1498abc37d3bf384aac7d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/4c5afdd345f1498abc37d3bf384aac7d 2024-12-06T06:15:43,615 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8daa628fabbb43749ea6793d7fbd268d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8daa628fabbb43749ea6793d7fbd268d 2024-12-06T06:15:43,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/b68ac8e30054446f981f219f9ad205a6 is 50, key is test_row_0/B:col10/1733465741858/Put/seqid=0 2024-12-06T06:15:43,616 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/457dae492e994fcd8100d7f65039d753 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/457dae492e994fcd8100d7f65039d753 2024-12-06T06:15:43,619 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/cb065e658d104b169dd43e986227d629 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/cb065e658d104b169dd43e986227d629 2024-12-06T06:15:43,620 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e92803818dd40029e5eec278dde3f59 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8e92803818dd40029e5eec278dde3f59 2024-12-06T06:15:43,621 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/ea61a1aecc5b49e0babb3c540f89c0aa to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/ea61a1aecc5b49e0babb3c540f89c0aa 2024-12-06T06:15:43,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742294_1470 (size=12301) 2024-12-06T06:15:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:43,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465803871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465803874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:43,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465803885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465803993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465803994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465804004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,029 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/b68ac8e30054446f981f219f9ad205a6 2024-12-06T06:15:44,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/39a864e3496040e399d1e113aceefad8 is 50, key is test_row_0/C:col10/1733465741858/Put/seqid=0 2024-12-06T06:15:44,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742295_1471 (size=12301) 2024-12-06T06:15:44,041 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/39a864e3496040e399d1e113aceefad8 2024-12-06T06:15:44,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/c3919d3e826f44f1862eea1b72115200 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/c3919d3e826f44f1862eea1b72115200 2024-12-06T06:15:44,048 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/c3919d3e826f44f1862eea1b72115200, entries=150, sequenceid=280, filesize=30.5 K 2024-12-06T06:15:44,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/b68ac8e30054446f981f219f9ad205a6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b68ac8e30054446f981f219f9ad205a6 2024-12-06T06:15:44,051 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b68ac8e30054446f981f219f9ad205a6, entries=150, sequenceid=280, filesize=12.0 K 2024-12-06T06:15:44,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/39a864e3496040e399d1e113aceefad8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/39a864e3496040e399d1e113aceefad8 2024-12-06T06:15:44,056 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/39a864e3496040e399d1e113aceefad8, entries=150, sequenceid=280, filesize=12.0 K 2024-12-06T06:15:44,057 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9d97d8dc224d0a067fc9941f0b9f555b in 873ms, sequenceid=280, compaction requested=false 2024-12-06T06:15:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-06T06:15:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-06T06:15:44,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-06T06:15:44,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-12-06T06:15:44,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8050 sec 2024-12-06T06:15:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:44,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:15:44,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:44,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:44,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:44,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:44,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:44,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:44,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120693d09f73de9845a896d2caf8b5230ecb_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465744061/Put/seqid=0 2024-12-06T06:15:44,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742296_1472 (size=17534) 2024-12-06T06:15:44,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465804128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465804143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465804214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465804214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465804223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465804254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465804254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T06:15:44,403 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:44,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-06T06:15:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T06:15:44,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:44,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:44,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465804474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465804474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:44,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120693d09f73de9845a896d2caf8b5230ecb_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120693d09f73de9845a896d2caf8b5230ecb_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:44,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/76b4e78e3b2641f4b69d0538d2d52c40, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:44,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/76b4e78e3b2641f4b69d0538d2d52c40 is 175, key is test_row_0/A:col10/1733465744061/Put/seqid=0 2024-12-06T06:15:44,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742297_1473 (size=48639) 2024-12-06T06:15:44,491 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/76b4e78e3b2641f4b69d0538d2d52c40 2024-12-06T06:15:44,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/4486536a9a544c5dab36b82a508a415c is 50, key is test_row_0/B:col10/1733465744061/Put/seqid=0 2024-12-06T06:15:44,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742298_1474 (size=12301) 2024-12-06T06:15:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T06:15:44,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465804534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465804534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465804544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T06:15:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:44,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T06:15:44,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:44,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:44,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:44,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T06:15:44,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465804784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465804784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:44,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T06:15:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:44,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:44,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/4486536a9a544c5dab36b82a508a415c 2024-12-06T06:15:44,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/54dd17f7f9194248871218688f865cdc is 50, key is test_row_0/C:col10/1733465744061/Put/seqid=0 2024-12-06T06:15:44,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742299_1475 (size=12301) 2024-12-06T06:15:45,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T06:15:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T06:15:45,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56592 deadline: 1733465805043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:45,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56584 deadline: 1733465805054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56580 deadline: 1733465805054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:45,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T06:15:45,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:45,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56562 deadline: 1733465805293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:45,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56564 deadline: 1733465805304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:15:45,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/54dd17f7f9194248871218688f865cdc 2024-12-06T06:15:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/76b4e78e3b2641f4b69d0538d2d52c40 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/76b4e78e3b2641f4b69d0538d2d52c40 2024-12-06T06:15:45,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/76b4e78e3b2641f4b69d0538d2d52c40, entries=250, sequenceid=297, filesize=47.5 K 2024-12-06T06:15:45,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/4486536a9a544c5dab36b82a508a415c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/4486536a9a544c5dab36b82a508a415c 2024-12-06T06:15:45,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T06:15:45,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:45,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/4486536a9a544c5dab36b82a508a415c, entries=150, sequenceid=297, filesize=12.0 K 2024-12-06T06:15:45,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/54dd17f7f9194248871218688f865cdc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/54dd17f7f9194248871218688f865cdc 2024-12-06T06:15:45,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/54dd17f7f9194248871218688f865cdc, entries=150, sequenceid=297, filesize=12.0 K 2024-12-06T06:15:45,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9d97d8dc224d0a067fc9941f0b9f555b in 1271ms, sequenceid=297, compaction requested=true 2024-12-06T06:15:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:45,333 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d97d8dc224d0a067fc9941f0b9f555b:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:45,333 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:45,334 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:45,334 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111595 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:45,334 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/B is initiating minor compaction (all files) 2024-12-06T06:15:45,334 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/A is initiating minor compaction (all files) 2024-12-06T06:15:45,334 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/B in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,334 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/A in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,334 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/7dd83579828344a5a4833715952c3887, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b68ac8e30054446f981f219f9ad205a6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/4486536a9a544c5dab36b82a508a415c] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=36.5 K 2024-12-06T06:15:45,334 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/6b20e04c51224083b859f3c9a9b4ef01, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/c3919d3e826f44f1862eea1b72115200, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/76b4e78e3b2641f4b69d0538d2d52c40] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=109.0 K 2024-12-06T06:15:45,334 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,335 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/6b20e04c51224083b859f3c9a9b4ef01, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/c3919d3e826f44f1862eea1b72115200, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/76b4e78e3b2641f4b69d0538d2d52c40] 2024-12-06T06:15:45,335 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd83579828344a5a4833715952c3887, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733465739525 2024-12-06T06:15:45,335 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b20e04c51224083b859f3c9a9b4ef01, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733465739525 2024-12-06T06:15:45,336 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b68ac8e30054446f981f219f9ad205a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733465741844 2024-12-06T06:15:45,336 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3919d3e826f44f1862eea1b72115200, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733465741844 2024-12-06T06:15:45,336 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4486536a9a544c5dab36b82a508a415c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733465743870 2024-12-06T06:15:45,336 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b4e78e3b2641f4b69d0538d2d52c40, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733465743870 2024-12-06T06:15:45,343 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:45,343 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/d53e5ecac978480ca8a354af63c5c508 is 50, key is test_row_0/B:col10/1733465744061/Put/seqid=0 2024-12-06T06:15:45,344 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742300_1476 (size=12404) 2024-12-06T06:15:45,346 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412063942519ef64843c1a0c72d5b7cf6ed2c_9d97d8dc224d0a067fc9941f0b9f555b store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:45,348 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412063942519ef64843c1a0c72d5b7cf6ed2c_9d97d8dc224d0a067fc9941f0b9f555b, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:45,348 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063942519ef64843c1a0c72d5b7cf6ed2c_9d97d8dc224d0a067fc9941f0b9f555b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:45,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742301_1477 (size=4469) 2024-12-06T06:15:45,353 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#A#compaction#405 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:45,353 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/e80ba7f85f47499fa7f28896b32c6785 is 175, key is test_row_0/A:col10/1733465744061/Put/seqid=0 2024-12-06T06:15:45,355 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/d53e5ecac978480ca8a354af63c5c508 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/d53e5ecac978480ca8a354af63c5c508 2024-12-06T06:15:45,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742302_1478 (size=31461) 2024-12-06T06:15:45,362 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/e80ba7f85f47499fa7f28896b32c6785 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/e80ba7f85f47499fa7f28896b32c6785 2024-12-06T06:15:45,362 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/B of 9d97d8dc224d0a067fc9941f0b9f555b into d53e5ecac978480ca8a354af63c5c508(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:45,362 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:45,362 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/B, priority=13, startTime=1733465745333; duration=0sec 2024-12-06T06:15:45,362 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:45,362 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:B 2024-12-06T06:15:45,362 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:15:45,363 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:15:45,363 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 9d97d8dc224d0a067fc9941f0b9f555b/C is initiating minor compaction (all files) 2024-12-06T06:15:45,363 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d97d8dc224d0a067fc9941f0b9f555b/C in TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,363 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8a3658362ada43e6b3b6f0793e489b7c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/39a864e3496040e399d1e113aceefad8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/54dd17f7f9194248871218688f865cdc] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp, totalSize=36.5 K 2024-12-06T06:15:45,363 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a3658362ada43e6b3b6f0793e489b7c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733465739525 2024-12-06T06:15:45,364 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 39a864e3496040e399d1e113aceefad8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733465741844 2024-12-06T06:15:45,364 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 54dd17f7f9194248871218688f865cdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733465743870 2024-12-06T06:15:45,366 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/A of 9d97d8dc224d0a067fc9941f0b9f555b into e80ba7f85f47499fa7f28896b32c6785(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:45,366 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:45,366 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/A, priority=13, startTime=1733465745333; duration=0sec 2024-12-06T06:15:45,366 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:45,366 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:A 2024-12-06T06:15:45,369 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d97d8dc224d0a067fc9941f0b9f555b#C#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:45,370 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/edba8ad72f6749a7a76a9405f3e3ce57 is 50, key is test_row_0/C:col10/1733465744061/Put/seqid=0 2024-12-06T06:15:45,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742303_1479 (size=12404) 2024-12-06T06:15:45,447 INFO [Thread-1805 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1805. 2024-12-06T06:15:45,447 DEBUG [Thread-1805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:45,447 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,447 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:45,449 INFO [Thread-1801 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1801. 2024-12-06T06:15:45,449 DEBUG [Thread-1801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:45,449 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,449 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:45,451 INFO [Thread-1799 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1799. 2024-12-06T06:15:45,451 DEBUG [Thread-1799 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:45,451 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,451 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:45,456 INFO [Thread-1803 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1803. 2024-12-06T06:15:45,456 DEBUG [Thread-1803 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:45,456 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,456 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:45,456 INFO [Thread-1807 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1807. 2024-12-06T06:15:45,456 DEBUG [Thread-1807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:45,456 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,457 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:45,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:45,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T06:15:45,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:45,478 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:15:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:45,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206eb2a1486a9124c9f9d20880158788393_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465744112/Put/seqid=0 2024-12-06T06:15:45,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742304_1480 (size=12454) 2024-12-06T06:15:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T06:15:45,778 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/edba8ad72f6749a7a76a9405f3e3ce57 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/edba8ad72f6749a7a76a9405f3e3ce57 2024-12-06T06:15:45,782 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d97d8dc224d0a067fc9941f0b9f555b/C of 9d97d8dc224d0a067fc9941f0b9f555b into edba8ad72f6749a7a76a9405f3e3ce57(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:45,782 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:45,782 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., storeName=9d97d8dc224d0a067fc9941f0b9f555b/C, priority=13, startTime=1733465745333; duration=0sec 2024-12-06T06:15:45,782 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:45,782 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d97d8dc224d0a067fc9941f0b9f555b:C 2024-12-06T06:15:45,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:45,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206eb2a1486a9124c9f9d20880158788393_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eb2a1486a9124c9f9d20880158788393_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:45,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/2ea17ce51c164f26b20c020992ba46bd, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:45,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/2ea17ce51c164f26b20c020992ba46bd is 175, key is test_row_0/A:col10/1733465744112/Put/seqid=0 2024-12-06T06:15:45,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742305_1481 (size=31255) 2024-12-06T06:15:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:46,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. as already flushing 2024-12-06T06:15:46,055 INFO [Thread-1788 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1788. 2024-12-06T06:15:46,056 DEBUG [Thread-1788 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:46,056 DEBUG [Thread-1788 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,056 DEBUG [Thread-1788 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:46,065 INFO [Thread-1790 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1790. 2024-12-06T06:15:46,065 DEBUG [Thread-1790 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:46,065 DEBUG [Thread-1790 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,065 DEBUG [Thread-1790 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:46,074 INFO [Thread-1792 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1792. 2024-12-06T06:15:46,074 DEBUG [Thread-1792 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:46,074 DEBUG [Thread-1792 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,074 DEBUG [Thread-1792 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:46,295 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/2ea17ce51c164f26b20c020992ba46bd 2024-12-06T06:15:46,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/3fb9c7ed11c348d3800e67b5d6f271a0 is 50, key is test_row_0/B:col10/1733465744112/Put/seqid=0 2024-12-06T06:15:46,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742306_1482 (size=12301) 2024-12-06T06:15:46,315 INFO [Thread-1794 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1794. 2024-12-06T06:15:46,315 INFO [Thread-1796 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1796. 2024-12-06T06:15:46,315 DEBUG [Thread-1794 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:46,315 DEBUG [Thread-1794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,315 DEBUG [Thread-1796 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:15:46,315 DEBUG [Thread-1796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,315 DEBUG [Thread-1796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,315 DEBUG [Thread-1794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:46,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:46,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T06:15:46,704 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/3fb9c7ed11c348d3800e67b5d6f271a0 2024-12-06T06:15:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/f44a981e6e274c7294300fe95cb925c4 is 50, key is test_row_0/C:col10/1733465744112/Put/seqid=0 2024-12-06T06:15:46,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742307_1483 (size=12301) 2024-12-06T06:15:47,114 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/f44a981e6e274c7294300fe95cb925c4 2024-12-06T06:15:47,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/2ea17ce51c164f26b20c020992ba46bd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2ea17ce51c164f26b20c020992ba46bd 2024-12-06T06:15:47,121 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2ea17ce51c164f26b20c020992ba46bd, entries=150, sequenceid=317, filesize=30.5 K 2024-12-06T06:15:47,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/3fb9c7ed11c348d3800e67b5d6f271a0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/3fb9c7ed11c348d3800e67b5d6f271a0 2024-12-06T06:15:47,124 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/3fb9c7ed11c348d3800e67b5d6f271a0, entries=150, sequenceid=317, filesize=12.0 K 2024-12-06T06:15:47,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/f44a981e6e274c7294300fe95cb925c4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/f44a981e6e274c7294300fe95cb925c4 2024-12-06T06:15:47,128 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/f44a981e6e274c7294300fe95cb925c4, entries=150, sequenceid=317, filesize=12.0 K 2024-12-06T06:15:47,129 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=33.54 KB/34350 for 9d97d8dc224d0a067fc9941f0b9f555b in 1650ms, sequenceid=317, compaction requested=false 2024-12-06T06:15:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 9d97d8dc224d0a067fc9941f0b9f555b: 2024-12-06T06:15:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-06T06:15:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-06T06:15:47,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-06T06:15:47,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7240 sec 2024-12-06T06:15:47,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.7280 sec 2024-12-06T06:15:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T06:15:48,564 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 895 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2676 rows 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 893 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2678 rows 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 889 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2664 rows 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 891 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2668 rows 2024-12-06T06:15:48,564 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 886 2024-12-06T06:15:48,565 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2658 rows 2024-12-06T06:15:48,565 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T06:15:48,565 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:15:48,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:48,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:48,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:15:48,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T06:15:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T06:15:48,568 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465748568"}]},"ts":"1733465748568"} 2024-12-06T06:15:48,570 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T06:15:48,570 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T06:15:48,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:15:48,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, UNASSIGN}] 2024-12-06T06:15:48,572 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, UNASSIGN 2024-12-06T06:15:48,572 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=9d97d8dc224d0a067fc9941f0b9f555b, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:48,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, UNASSIGN because future has completed 2024-12-06T06:15:48,574 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:15:48,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:15:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T06:15:48,725 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:48,725 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 9d97d8dc224d0a067fc9941f0b9f555b, disabling compactions & flushes 2024-12-06T06:15:48,726 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. after waiting 0 ms 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:48,726 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 9d97d8dc224d0a067fc9941f0b9f555b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=A 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=B 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d97d8dc224d0a067fc9941f0b9f555b, store=C 2024-12-06T06:15:48,726 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:48,731 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f821d0fb2fb241239dfa743c5684b2b2_9d97d8dc224d0a067fc9941f0b9f555b is 50, key is test_row_0/A:col10/1733465746073/Put/seqid=0 2024-12-06T06:15:48,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742308_1484 (size=12454) 2024-12-06T06:15:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T06:15:49,134 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:49,138 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f821d0fb2fb241239dfa743c5684b2b2_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f821d0fb2fb241239dfa743c5684b2b2_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:49,138 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/999e376aed12406b879ac4481df0c2b3, store: [table=TestAcidGuarantees family=A region=9d97d8dc224d0a067fc9941f0b9f555b] 2024-12-06T06:15:49,139 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/999e376aed12406b879ac4481df0c2b3 is 175, key is test_row_0/A:col10/1733465746073/Put/seqid=0 2024-12-06T06:15:49,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742309_1485 (size=31255) 2024-12-06T06:15:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T06:15:49,543 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/999e376aed12406b879ac4481df0c2b3 2024-12-06T06:15:49,548 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/9e7734e037844084b1ca3fe6373e91f2 is 50, key is test_row_0/B:col10/1733465746073/Put/seqid=0 2024-12-06T06:15:49,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742310_1486 (size=12301) 2024-12-06T06:15:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T06:15:49,952 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/9e7734e037844084b1ca3fe6373e91f2 2024-12-06T06:15:49,957 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/0c2ebbcbc1ee4bc6add240df67a9d8dd is 50, key is test_row_0/C:col10/1733465746073/Put/seqid=0 2024-12-06T06:15:49,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742311_1487 (size=12301) 2024-12-06T06:15:50,361 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/0c2ebbcbc1ee4bc6add240df67a9d8dd 2024-12-06T06:15:50,364 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/A/999e376aed12406b879ac4481df0c2b3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/999e376aed12406b879ac4481df0c2b3 2024-12-06T06:15:50,366 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/999e376aed12406b879ac4481df0c2b3, entries=150, sequenceid=326, filesize=30.5 K 2024-12-06T06:15:50,367 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/B/9e7734e037844084b1ca3fe6373e91f2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/9e7734e037844084b1ca3fe6373e91f2 2024-12-06T06:15:50,369 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/9e7734e037844084b1ca3fe6373e91f2, entries=150, sequenceid=326, filesize=12.0 K 2024-12-06T06:15:50,370 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/.tmp/C/0c2ebbcbc1ee4bc6add240df67a9d8dd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0c2ebbcbc1ee4bc6add240df67a9d8dd 2024-12-06T06:15:50,373 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0c2ebbcbc1ee4bc6add240df67a9d8dd, entries=150, sequenceid=326, filesize=12.0 K 2024-12-06T06:15:50,374 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9d97d8dc224d0a067fc9941f0b9f555b in 1648ms, sequenceid=326, compaction requested=true 2024-12-06T06:15:50,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/d213a87eff9a49dc8aa313d3ba3de37d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/24e65b91569e4df6a74363e1aa1e2abe, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/0d6d0288c9594a328d6071652e313410, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/6b20e04c51224083b859f3c9a9b4ef01, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/c3919d3e826f44f1862eea1b72115200, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/76b4e78e3b2641f4b69d0538d2d52c40] to archive 2024-12-06T06:15:50,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:50,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/d213a87eff9a49dc8aa313d3ba3de37d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/d213a87eff9a49dc8aa313d3ba3de37d 2024-12-06T06:15:50,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/24e65b91569e4df6a74363e1aa1e2abe to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/24e65b91569e4df6a74363e1aa1e2abe 2024-12-06T06:15:50,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/0d6d0288c9594a328d6071652e313410 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/0d6d0288c9594a328d6071652e313410 2024-12-06T06:15:50,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/6b20e04c51224083b859f3c9a9b4ef01 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/6b20e04c51224083b859f3c9a9b4ef01 2024-12-06T06:15:50,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/c3919d3e826f44f1862eea1b72115200 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/c3919d3e826f44f1862eea1b72115200 2024-12-06T06:15:50,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/76b4e78e3b2641f4b69d0538d2d52c40 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/76b4e78e3b2641f4b69d0538d2d52c40 2024-12-06T06:15:50,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/7dd83579828344a5a4833715952c3887, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b68ac8e30054446f981f219f9ad205a6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/4486536a9a544c5dab36b82a508a415c] to archive 2024-12-06T06:15:50,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:50,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/7dd83579828344a5a4833715952c3887 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/7dd83579828344a5a4833715952c3887 2024-12-06T06:15:50,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b68ac8e30054446f981f219f9ad205a6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/b68ac8e30054446f981f219f9ad205a6 2024-12-06T06:15:50,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/4486536a9a544c5dab36b82a508a415c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/4486536a9a544c5dab36b82a508a415c 2024-12-06T06:15:50,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8a3658362ada43e6b3b6f0793e489b7c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/39a864e3496040e399d1e113aceefad8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/54dd17f7f9194248871218688f865cdc] to archive 2024-12-06T06:15:50,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:15:50,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8a3658362ada43e6b3b6f0793e489b7c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/8a3658362ada43e6b3b6f0793e489b7c 2024-12-06T06:15:50,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/39a864e3496040e399d1e113aceefad8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/39a864e3496040e399d1e113aceefad8 2024-12-06T06:15:50,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/54dd17f7f9194248871218688f865cdc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/54dd17f7f9194248871218688f865cdc 2024-12-06T06:15:50,390 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/recovered.edits/329.seqid, newMaxSeqId=329, maxSeqId=4 2024-12-06T06:15:50,391 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. 2024-12-06T06:15:50,391 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 9d97d8dc224d0a067fc9941f0b9f555b: Waiting for close lock at 1733465748726Running coprocessor pre-close hooks at 1733465748726Disabling compacts and flushes for region at 1733465748726Disabling writes for close at 1733465748726Obtaining lock to block concurrent updates at 1733465748726Preparing flush snapshotting stores in 9d97d8dc224d0a067fc9941f0b9f555b at 1733465748726Finished memstore snapshotting TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733465748726Flushing stores of TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b. at 1733465748727 (+1 ms)Flushing 9d97d8dc224d0a067fc9941f0b9f555b/A: creating writer at 1733465748727Flushing mob file 9d97d8dc224d0a067fc9941f0b9f555b/A: appending metadata at 1733465748731 (+4 ms)Flushing mob file 9d97d8dc224d0a067fc9941f0b9f555b/A: closing flushed file at 1733465748731Flushing 9d97d8dc224d0a067fc9941f0b9f555b/A: appending metadata at 1733465749138 (+407 ms)Flushing 9d97d8dc224d0a067fc9941f0b9f555b/A: closing flushed file at 1733465749139 (+1 ms)Flushing 9d97d8dc224d0a067fc9941f0b9f555b/B: creating writer at 1733465749546 (+407 ms)Flushing 9d97d8dc224d0a067fc9941f0b9f555b/B: appending metadata at 1733465749548 (+2 ms)Flushing 9d97d8dc224d0a067fc9941f0b9f555b/B: closing flushed file at 1733465749548Flushing 9d97d8dc224d0a067fc9941f0b9f555b/C: creating writer at 1733465749955 (+407 ms)Flushing 9d97d8dc224d0a067fc9941f0b9f555b/C: appending metadata at 1733465749957 (+2 ms)Flushing 9d97d8dc224d0a067fc9941f0b9f555b/C: closing flushed file at 1733465749957Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@123fbe8d: reopening flushed file at 1733465750363 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ae376a8: reopening flushed file at 1733465750366 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30c6b27: reopening flushed file at 1733465750369 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9d97d8dc224d0a067fc9941f0b9f555b in 1648ms, sequenceid=326, compaction requested=true at 1733465750374 (+5 ms)Writing region close event to WAL at 1733465750388 (+14 ms)Running coprocessor post-close hooks at 1733465750391 (+3 ms)Closed at 1733465750391 2024-12-06T06:15:50,392 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,393 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=9d97d8dc224d0a067fc9941f0b9f555b, regionState=CLOSED 2024-12-06T06:15:50,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:15:50,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-06T06:15:50,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d97d8dc224d0a067fc9941f0b9f555b, server=6336e8474e1c,39987,1733465620995 in 1.8210 sec 2024-12-06T06:15:50,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-06T06:15:50,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d97d8dc224d0a067fc9941f0b9f555b, UNASSIGN in 1.8250 sec 2024-12-06T06:15:50,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-06T06:15:50,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8270 sec 2024-12-06T06:15:50,399 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465750399"}]},"ts":"1733465750399"} 2024-12-06T06:15:50,401 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T06:15:50,401 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T06:15:50,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8350 sec 2024-12-06T06:15:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T06:15:50,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T06:15:50,714 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:50,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T06:15:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:50,715 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T06:15:50,716 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:50,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,720 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/recovered.edits] 2024-12-06T06:15:50,722 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2ea17ce51c164f26b20c020992ba46bd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/2ea17ce51c164f26b20c020992ba46bd 2024-12-06T06:15:50,722 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/999e376aed12406b879ac4481df0c2b3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/999e376aed12406b879ac4481df0c2b3 2024-12-06T06:15:50,723 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/e80ba7f85f47499fa7f28896b32c6785 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/A/e80ba7f85f47499fa7f28896b32c6785 2024-12-06T06:15:50,725 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/3fb9c7ed11c348d3800e67b5d6f271a0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/3fb9c7ed11c348d3800e67b5d6f271a0 2024-12-06T06:15:50,726 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/9e7734e037844084b1ca3fe6373e91f2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/9e7734e037844084b1ca3fe6373e91f2 2024-12-06T06:15:50,727 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/d53e5ecac978480ca8a354af63c5c508 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/B/d53e5ecac978480ca8a354af63c5c508 2024-12-06T06:15:50,728 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0c2ebbcbc1ee4bc6add240df67a9d8dd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/0c2ebbcbc1ee4bc6add240df67a9d8dd 2024-12-06T06:15:50,729 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/edba8ad72f6749a7a76a9405f3e3ce57 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/edba8ad72f6749a7a76a9405f3e3ce57 2024-12-06T06:15:50,730 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/f44a981e6e274c7294300fe95cb925c4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/C/f44a981e6e274c7294300fe95cb925c4 2024-12-06T06:15:50,731 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/recovered.edits/329.seqid to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b/recovered.edits/329.seqid 2024-12-06T06:15:50,732 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,732 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T06:15:50,732 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T06:15:50,733 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T06:15:50,734 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065c48cbb11a284dfdb6d5294d03fff4b5_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065c48cbb11a284dfdb6d5294d03fff4b5_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,735 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206806997ed2297422596123c3832ccacba_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206806997ed2297422596123c3832ccacba_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,736 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120693d09f73de9845a896d2caf8b5230ecb_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120693d09f73de9845a896d2caf8b5230ecb_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,737 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120699b658054f3d4402925396b02edf026b_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120699b658054f3d4402925396b02edf026b_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,738 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069b8e868bf70f4a3aa4c74da577d99bd6_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069b8e868bf70f4a3aa4c74da577d99bd6_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,738 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a0d9e851755a478c9da04ce76a50fd83_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a0d9e851755a478c9da04ce76a50fd83_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,739 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a1f4bf98794548c3b3da7b7a4880f657_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a1f4bf98794548c3b3da7b7a4880f657_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,740 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a96f8b28b7974709a40cc8b6e1a076b7_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a96f8b28b7974709a40cc8b6e1a076b7_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,741 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ae092c2648af490f95b33a15d7823b62_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ae092c2648af490f95b33a15d7823b62_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,742 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206afcd80f58f2b4cf4a30c54d3e37b0520_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206afcd80f58f2b4cf4a30c54d3e37b0520_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,742 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ca3014f2212f415b9d48b5d2407625a7_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ca3014f2212f415b9d48b5d2407625a7_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,743 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cc039f9a453840d0b3156571512c2d19_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cc039f9a453840d0b3156571512c2d19_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,744 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d0f03865870b44aeb703b3dc2fe26bda_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d0f03865870b44aeb703b3dc2fe26bda_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,745 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eb2a1486a9124c9f9d20880158788393_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eb2a1486a9124c9f9d20880158788393_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,746 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f821d0fb2fb241239dfa743c5684b2b2_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f821d0fb2fb241239dfa743c5684b2b2_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,746 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fbdda64d1168450e8d3120ef8dd44309_9d97d8dc224d0a067fc9941f0b9f555b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fbdda64d1168450e8d3120ef8dd44309_9d97d8dc224d0a067fc9941f0b9f555b 2024-12-06T06:15:50,747 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T06:15:50,748 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:50,750 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T06:15:50,752 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T06:15:50,753 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:50,753 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T06:15:50,753 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733465750753"}]},"ts":"9223372036854775807"} 2024-12-06T06:15:50,754 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T06:15:50,754 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9d97d8dc224d0a067fc9941f0b9f555b, NAME => 'TestAcidGuarantees,,1733465722342.9d97d8dc224d0a067fc9941f0b9f555b.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T06:15:50,754 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T06:15:50,755 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733465750754"}]},"ts":"9223372036854775807"} 2024-12-06T06:15:50,756 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T06:15:50,756 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:50,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-12-06T06:15:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T06:15:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T06:15:50,825 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:50,834 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 246) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869409647_22 at /127.0.0.1:40310 [Waiting for operation #946] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869409647_22 at /127.0.0.1:51192 [Waiting for operation #842] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/6336e8474e1c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2037682797_22 at /127.0.0.1:51202 [Waiting for operation #829] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2037682797_22 at /127.0.0.1:51230 [Waiting for operation #238] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=342 (was 404), ProcessCount=11 (was 11), AvailableMemoryMB=4321 (was 4825) 2024-12-06T06:15:50,843 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=4321 2024-12-06T06:15:50,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:15:50,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:15:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T06:15:50,847 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T06:15:50,847 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:50,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-12-06T06:15:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-06T06:15:50,847 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T06:15:50,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742312_1488 (size=1061) 2024-12-06T06:15:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-06T06:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-06T06:15:51,254 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea 2024-12-06T06:15:51,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742313_1489 (size=53) 2024-12-06T06:15:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-06T06:15:51,659 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:15:51,659 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6da2c4ed30e0a36b951d14c383b1d513, disabling compactions & flushes 2024-12-06T06:15:51,659 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:51,659 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:51,659 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. after waiting 0 ms 2024-12-06T06:15:51,659 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:51,659 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:51,659 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6da2c4ed30e0a36b951d14c383b1d513: Waiting for close lock at 1733465751659Disabling compacts and flushes for region at 1733465751659Disabling writes for close at 1733465751659Writing region close event to WAL at 1733465751659Closed at 1733465751659 2024-12-06T06:15:51,660 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T06:15:51,661 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733465751660"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733465751660"}]},"ts":"1733465751660"} 2024-12-06T06:15:51,662 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T06:15:51,662 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T06:15:51,663 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465751662"}]},"ts":"1733465751662"} 2024-12-06T06:15:51,663 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T06:15:51,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, ASSIGN}] 2024-12-06T06:15:51,665 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, ASSIGN 2024-12-06T06:15:51,665 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, ASSIGN; state=OFFLINE, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=false 2024-12-06T06:15:51,816 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=6da2c4ed30e0a36b951d14c383b1d513, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:51,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, ASSIGN because future has completed 2024-12-06T06:15:51,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:15:51,972 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:51,972 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:15:51,973 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,973 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:15:51,973 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,973 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,974 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,975 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:51,975 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6da2c4ed30e0a36b951d14c383b1d513 columnFamilyName A 2024-12-06T06:15:51,975 DEBUG [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:51,976 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.HStore(327): Store=6da2c4ed30e0a36b951d14c383b1d513/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:51,976 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,976 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:51,977 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6da2c4ed30e0a36b951d14c383b1d513 columnFamilyName B 2024-12-06T06:15:51,977 DEBUG [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:51,977 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.HStore(327): Store=6da2c4ed30e0a36b951d14c383b1d513/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:51,977 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,978 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:15:51,978 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6da2c4ed30e0a36b951d14c383b1d513 columnFamilyName C 2024-12-06T06:15:51,978 DEBUG [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:15:51,978 INFO [StoreOpener-6da2c4ed30e0a36b951d14c383b1d513-1 {}] regionserver.HStore(327): Store=6da2c4ed30e0a36b951d14c383b1d513/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:15:51,978 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:51,978 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,979 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,979 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,980 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,980 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,980 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:15:51,981 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,982 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T06:15:51,983 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 6da2c4ed30e0a36b951d14c383b1d513; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59476153, jitterRate=-0.11373625695705414}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:15:51,983 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-06T06:15:51,983 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 6da2c4ed30e0a36b951d14c383b1d513: Running coprocessor pre-open hook at 1733465751973Writing region info on filesystem at 1733465751973Initializing all the Stores at 1733465751974 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465751974Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465751974Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465751974Cleaning up temporary data from old regions at 1733465751980 (+6 ms)Running coprocessor post-open hooks at 1733465751983 (+3 ms)Region opened successfully at 1733465751983 2024-12-06T06:15:51,984 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., pid=122, masterSystemTime=1733465751969 2024-12-06T06:15:51,985 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:51,985 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:51,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=6da2c4ed30e0a36b951d14c383b1d513, regionState=OPEN, openSeqNum=2, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:51,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:15:51,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-06T06:15:51,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 in 171 msec 2024-12-06T06:15:51,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-12-06T06:15:51,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, ASSIGN in 325 msec 2024-12-06T06:15:51,991 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T06:15:51,991 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465751991"}]},"ts":"1733465751991"} 2024-12-06T06:15:51,993 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T06:15:51,993 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T06:15:51,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-12-06T06:15:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-06T06:15:52,994 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:52,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:52,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:52,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:52,996 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41111a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:52,997 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb056a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:52,998 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,000 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T06:15:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 6336e8474e1c,40959,1733465620259 2024-12-06T06:15:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@152e08cb 2024-12-06T06:15:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T06:15:53,002 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T06:15:53,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33765d1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51fdfa75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,006 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3c9f23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21da86e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,008 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4c937d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,009 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73cfe2ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,010 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ce5957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,012 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75197640, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f7b4aa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,015 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c39579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,016 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8329e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a2de97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,019 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b70e5e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb9618b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2915fa09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28092383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b31ef7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,024 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9b8d16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2638f346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:15:53,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:15:53,029 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff7bdb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:15:53,030 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:15:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46efc220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:15:53,030 DEBUG [Thread-2189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,030 DEBUG [Thread-2186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,031 DEBUG [Thread-2195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,031 DEBUG [Thread-2178 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,032 DEBUG [Thread-2191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,032 DEBUG [Thread-2197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:53,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,034 DEBUG [Thread-2180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-06T06:15:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T06:15:53,038 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:53,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,040 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:53,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,045 DEBUG [Thread-2184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,045 DEBUG [Thread-2182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,046 DEBUG [Thread-2193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:15:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:15:53,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-06T06:15:53,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:15:53,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:53,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:15:53,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:53,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:15:53,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:53,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:15:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2] 2024-12-06T06:15:53,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/cfe0c9b1a0664ed39f78e8fa713b98c9 is 50, key is test_row_0/A:col10/1733465753045/Put/seqid=0 2024-12-06T06:15:53,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742314_1490 (size=12001) 2024-12-06T06:15:53,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465753069, max timestamp=1733465753113Num uniques 859; 2024-12-06T06:15:53,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:53,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465753069, max timestamp=1733465753113Num uniques 859; 2024-12-06T06:15:53,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:53,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465753069, max timestamp=1733465753113Num uniques 859; 2024-12-06T06:15:53,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T06:15:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465813148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465813149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:53,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:53,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465813190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465813190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465813190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465813293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465813293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465813314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465813314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465813314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T06:15:53,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/cfe0c9b1a0664ed39f78e8fa713b98c9 2024-12-06T06:15:53,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/fc6fdacac8ff44fdbaef1b003e8b41d2 is 50, key is test_row_0/B:col10/1733465753045/Put/seqid=0 2024-12-06T06:15:53,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742315_1491 (size=12001) 2024-12-06T06:15:53,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465813514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465813514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465813533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465813533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465813533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T06:15:53,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:53,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:53,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465813834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465813834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465813853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465813853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465813854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:53,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/fc6fdacac8ff44fdbaef1b003e8b41d2 2024-12-06T06:15:53,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/6519c805741842c9b92b9f67be1e7040 is 50, key is test_row_0/C:col10/1733465753045/Put/seqid=0 2024-12-06T06:15:53,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742316_1492 (size=12001) 2024-12-06T06:15:53,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:53,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T06:15:54,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:54,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/6519c805741842c9b92b9f67be1e7040 2024-12-06T06:15:54,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/cfe0c9b1a0664ed39f78e8fa713b98c9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cfe0c9b1a0664ed39f78e8fa713b98c9 2024-12-06T06:15:54,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cfe0c9b1a0664ed39f78e8fa713b98c9, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T06:15:54,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/fc6fdacac8ff44fdbaef1b003e8b41d2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/fc6fdacac8ff44fdbaef1b003e8b41d2 2024-12-06T06:15:54,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/fc6fdacac8ff44fdbaef1b003e8b41d2, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T06:15:54,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/6519c805741842c9b92b9f67be1e7040 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/6519c805741842c9b92b9f67be1e7040 2024-12-06T06:15:54,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/6519c805741842c9b92b9f67be1e7040, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T06:15:54,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6da2c4ed30e0a36b951d14c383b1d513 in 1289ms, sequenceid=12, compaction requested=false 2024-12-06T06:15:54,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:15:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:54,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-06T06:15:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:15:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:15:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:15:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:54,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/84cb6038147840528dfc304b7a521ecf is 50, key is test_row_0/A:col10/1733465754354/Put/seqid=0 2024-12-06T06:15:54,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742317_1493 (size=14341) 2024-12-06T06:15:54,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465814374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465814374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465814376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465814386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465814386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:54,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:54,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465814493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465814493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465814493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465814704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465814713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465814713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:54,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/84cb6038147840528dfc304b7a521ecf 2024-12-06T06:15:54,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/edf43a6e96cb410892c3a103dc69393b is 50, key is test_row_0/B:col10/1733465754354/Put/seqid=0 2024-12-06T06:15:54,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742318_1494 (size=12001) 2024-12-06T06:15:54,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/edf43a6e96cb410892c3a103dc69393b 2024-12-06T06:15:54,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/4771616e65ef433198d627464db2055c is 50, key is test_row_0/C:col10/1733465754354/Put/seqid=0 2024-12-06T06:15:54,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742319_1495 (size=12001) 2024-12-06T06:15:54,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:54,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465815014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:55,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:55,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465815024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465815024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:55,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T06:15:55,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/4771616e65ef433198d627464db2055c 2024-12-06T06:15:55,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/84cb6038147840528dfc304b7a521ecf as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/84cb6038147840528dfc304b7a521ecf 2024-12-06T06:15:55,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/84cb6038147840528dfc304b7a521ecf, entries=200, sequenceid=50, filesize=14.0 K 2024-12-06T06:15:55,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/edf43a6e96cb410892c3a103dc69393b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/edf43a6e96cb410892c3a103dc69393b 2024-12-06T06:15:55,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/edf43a6e96cb410892c3a103dc69393b, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T06:15:55,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/4771616e65ef433198d627464db2055c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/4771616e65ef433198d627464db2055c 2024-12-06T06:15:55,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/4771616e65ef433198d627464db2055c, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T06:15:55,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 6da2c4ed30e0a36b951d14c383b1d513 in 848ms, sequenceid=50, compaction requested=false 2024-12-06T06:15:55,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:15:55,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T06:15:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:55,335 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-06T06:15:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:15:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:15:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:15:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:55,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/f48261375cd243dca4fa231446fc98ff is 50, key is test_row_1/A:col10/1733465754384/Put/seqid=0 2024-12-06T06:15:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742320_1496 (size=9657) 2024-12-06T06:15:55,346 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/f48261375cd243dca4fa231446fc98ff 2024-12-06T06:15:55,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/8f5fea87dac14a19ab72175b08ea0b08 is 50, key is test_row_1/B:col10/1733465754384/Put/seqid=0 2024-12-06T06:15:55,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742321_1497 (size=9657) 2024-12-06T06:15:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:55,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:55,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465755461, max timestamp=1733465755497Num uniques 859; 2024-12-06T06:15:55,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:55,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465755461, max timestamp=1733465755497Num uniques 859; 2024-12-06T06:15:55,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:55,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465755461, max timestamp=1733465755497Num uniques 859; 2024-12-06T06:15:55,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:15:55,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733465755546, max timestamp=1733465755564Num uniques 1018; 2024-12-06T06:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T06:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733465755546, max timestamp=1733465755564Num uniques 1018; 2024-12-06T06:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T06:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733465755546, max timestamp=1733465755564Num uniques 1018; 2024-12-06T06:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-06T06:15:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465815605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465815605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465815607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465815608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465815615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465815724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465815724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465815724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465815724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465815734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,757 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/8f5fea87dac14a19ab72175b08ea0b08 2024-12-06T06:15:55,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/7af225dad3f043b3a4a6c8a4a7ac2924 is 50, key is test_row_1/C:col10/1733465754384/Put/seqid=0 2024-12-06T06:15:55,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742322_1498 (size=9657) 2024-12-06T06:15:55,912 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T06:15:55,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465815943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465815943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465815944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465815944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:55,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465815954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,171 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/7af225dad3f043b3a4a6c8a4a7ac2924 2024-12-06T06:15:56,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/f48261375cd243dca4fa231446fc98ff as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f48261375cd243dca4fa231446fc98ff 2024-12-06T06:15:56,179 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f48261375cd243dca4fa231446fc98ff, entries=100, sequenceid=59, filesize=9.4 K 2024-12-06T06:15:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/8f5fea87dac14a19ab72175b08ea0b08 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/8f5fea87dac14a19ab72175b08ea0b08 2024-12-06T06:15:56,183 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/8f5fea87dac14a19ab72175b08ea0b08, entries=100, sequenceid=59, filesize=9.4 K 2024-12-06T06:15:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/7af225dad3f043b3a4a6c8a4a7ac2924 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7af225dad3f043b3a4a6c8a4a7ac2924 2024-12-06T06:15:56,186 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7af225dad3f043b3a4a6c8a4a7ac2924, entries=100, sequenceid=59, filesize=9.4 K 2024-12-06T06:15:56,187 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 6da2c4ed30e0a36b951d14c383b1d513 in 852ms, sequenceid=59, compaction requested=true 2024-12-06T06:15:56,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:15:56,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:56,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-06T06:15:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-06T06:15:56,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-06T06:15:56,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1480 sec 2024-12-06T06:15:56,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 3.1560 sec 2024-12-06T06:15:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:56,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-06T06:15:56,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:15:56,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:56,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:15:56,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:56,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:15:56,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:56,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/89d63734460d45688aae6c6739d1dec0 is 50, key is test_row_0/A:col10/1733465756267/Put/seqid=0 2024-12-06T06:15:56,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742323_1499 (size=16681) 2024-12-06T06:15:56,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465816273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465816275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465816282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465816290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465816291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465816394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465816394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465816394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465816403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465816413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465816604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465816614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465816614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465816614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465816633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/89d63734460d45688aae6c6739d1dec0 2024-12-06T06:15:56,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/ac3dccb8e1e142779f7b4822b844b2e7 is 50, key is test_row_0/B:col10/1733465756267/Put/seqid=0 2024-12-06T06:15:56,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742324_1500 (size=12001) 2024-12-06T06:15:56,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/ac3dccb8e1e142779f7b4822b844b2e7 2024-12-06T06:15:56,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/0843b41ed259461db1ba7d964e5b1389 is 50, key is test_row_0/C:col10/1733465756267/Put/seqid=0 2024-12-06T06:15:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742325_1501 (size=12001) 2024-12-06T06:15:56,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465816924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465816924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465816924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465816924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465816953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/0843b41ed259461db1ba7d964e5b1389 2024-12-06T06:15:57,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/89d63734460d45688aae6c6739d1dec0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/89d63734460d45688aae6c6739d1dec0 2024-12-06T06:15:57,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/89d63734460d45688aae6c6739d1dec0, entries=250, sequenceid=115, filesize=16.3 K 2024-12-06T06:15:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/ac3dccb8e1e142779f7b4822b844b2e7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/ac3dccb8e1e142779f7b4822b844b2e7 2024-12-06T06:15:57,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/ac3dccb8e1e142779f7b4822b844b2e7, entries=150, sequenceid=115, filesize=11.7 K 2024-12-06T06:15:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/0843b41ed259461db1ba7d964e5b1389 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/0843b41ed259461db1ba7d964e5b1389 2024-12-06T06:15:57,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/0843b41ed259461db1ba7d964e5b1389, entries=150, sequenceid=115, filesize=11.7 K 2024-12-06T06:15:57,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~439.88 KB/450432, currentSize=33.54 KB/34350 for 6da2c4ed30e0a36b951d14c383b1d513 in 868ms, sequenceid=115, compaction requested=true 2024-12-06T06:15:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:15:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:15:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:57,137 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:57,137 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:15:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:15:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:57,138 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:57,138 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:57,138 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/B is initiating minor compaction (all files) 2024-12-06T06:15:57,138 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/A is initiating minor compaction (all files) 2024-12-06T06:15:57,138 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/A in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:57,138 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/B in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:57,139 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cfe0c9b1a0664ed39f78e8fa713b98c9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/84cb6038147840528dfc304b7a521ecf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f48261375cd243dca4fa231446fc98ff, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/89d63734460d45688aae6c6739d1dec0] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=51.4 K 2024-12-06T06:15:57,139 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/fc6fdacac8ff44fdbaef1b003e8b41d2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/edf43a6e96cb410892c3a103dc69393b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/8f5fea87dac14a19ab72175b08ea0b08, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/ac3dccb8e1e142779f7b4822b844b2e7] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=44.6 K 2024-12-06T06:15:57,139 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe0c9b1a0664ed39f78e8fa713b98c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733465753042 2024-12-06T06:15:57,139 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting fc6fdacac8ff44fdbaef1b003e8b41d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733465753042 2024-12-06T06:15:57,139 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84cb6038147840528dfc304b7a521ecf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465753147 2024-12-06T06:15:57,139 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting edf43a6e96cb410892c3a103dc69393b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465753147 2024-12-06T06:15:57,139 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f48261375cd243dca4fa231446fc98ff, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733465754384 2024-12-06T06:15:57,139 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5fea87dac14a19ab72175b08ea0b08, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733465754384 2024-12-06T06:15:57,140 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d63734460d45688aae6c6739d1dec0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465755603 2024-12-06T06:15:57,140 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3dccb8e1e142779f7b4822b844b2e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465755605 2024-12-06T06:15:57,148 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#B#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:57,149 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/24e247e3a11b41bebb916075857343cf is 50, key is test_row_0/B:col10/1733465756267/Put/seqid=0 2024-12-06T06:15:57,152 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#A#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:57,153 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/c1b5669aa93f41c7b721db84da2e4805 is 50, key is test_row_0/A:col10/1733465756267/Put/seqid=0 2024-12-06T06:15:57,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742326_1502 (size=12139) 2024-12-06T06:15:57,161 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/24e247e3a11b41bebb916075857343cf as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/24e247e3a11b41bebb916075857343cf 2024-12-06T06:15:57,166 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/B of 6da2c4ed30e0a36b951d14c383b1d513 into 24e247e3a11b41bebb916075857343cf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:57,166 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:15:57,166 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/B, priority=12, startTime=1733465757137; duration=0sec 2024-12-06T06:15:57,167 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:15:57,167 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:B 2024-12-06T06:15:57,167 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:15:57,168 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:15:57,168 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/C is initiating minor compaction (all files) 2024-12-06T06:15:57,168 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/C in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:57,168 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/6519c805741842c9b92b9f67be1e7040, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/4771616e65ef433198d627464db2055c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7af225dad3f043b3a4a6c8a4a7ac2924, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/0843b41ed259461db1ba7d964e5b1389] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=44.6 K 2024-12-06T06:15:57,169 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6519c805741842c9b92b9f67be1e7040, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733465753042 2024-12-06T06:15:57,169 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4771616e65ef433198d627464db2055c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465753147 2024-12-06T06:15:57,169 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af225dad3f043b3a4a6c8a4a7ac2924, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733465754384 2024-12-06T06:15:57,169 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0843b41ed259461db1ba7d964e5b1389, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465755605 2024-12-06T06:15:57,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742327_1503 (size=12139) 2024-12-06T06:15:57,178 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:15:57,179 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/c62f735c2fc840d4951d8fbe5eae3737 is 50, key is test_row_0/C:col10/1733465756267/Put/seqid=0 2024-12-06T06:15:57,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742328_1504 (size=12139) 2024-12-06T06:15:57,192 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/c62f735c2fc840d4951d8fbe5eae3737 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/c62f735c2fc840d4951d8fbe5eae3737 2024-12-06T06:15:57,196 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/C of 6da2c4ed30e0a36b951d14c383b1d513 into c62f735c2fc840d4951d8fbe5eae3737(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:57,196 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:15:57,196 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/C, priority=12, startTime=1733465757137; duration=0sec 2024-12-06T06:15:57,196 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:57,196 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:C 2024-12-06T06:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T06:15:57,204 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:57,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-06T06:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T06:15:57,206 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:57,207 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:57,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T06:15:57,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T06:15:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:57,359 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T06:15:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:15:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:15:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:15:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:15:57,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/e052fb7f9f8642fcaaa4dda0eda31ba1 is 50, key is test_row_0/A:col10/1733465756288/Put/seqid=0 2024-12-06T06:15:57,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742329_1505 (size=9657) 2024-12-06T06:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:57,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:57,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465757467, max timestamp=1733465757485Num uniques 859; 2024-12-06T06:15:57,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:15:57,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465757467, max timestamp=1733465757485Num uniques 859; 2024-12-06T06:15:57,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:15:57,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465757467, max timestamp=1733465757485Num uniques 859; 2024-12-06T06:15:57,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:15:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T06:15:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733465757504, max timestamp=1733465757525Num uniques 1068; 2024-12-06T06:15:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-06T06:15:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733465757504, max timestamp=1733465757525Num uniques 1068; 2024-12-06T06:15:57,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-06T06:15:57,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733465757504, max timestamp=1733465757525Num uniques 1068; 2024-12-06T06:15:57,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-06T06:15:57,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465817569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,577 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/c1b5669aa93f41c7b721db84da2e4805 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c1b5669aa93f41c7b721db84da2e4805 2024-12-06T06:15:57,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733465757556, max timestamp=1733465757573Num uniques 1177; 2024-12-06T06:15:57,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-06T06:15:57,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733465757556, max timestamp=1733465757573Num uniques 1177; 2024-12-06T06:15:57,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-06T06:15:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733465757556, max timestamp=1733465757573Num uniques 1177; 2024-12-06T06:15:57,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-06T06:15:57,581 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/A of 6da2c4ed30e0a36b951d14c383b1d513 into c1b5669aa93f41c7b721db84da2e4805(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:15:57,581 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:15:57,581 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/A, priority=12, startTime=1733465757137; duration=0sec 2024-12-06T06:15:57,581 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:15:57,581 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:A 2024-12-06T06:15:57,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465817614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465817622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465817623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465817626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465817684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465817733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465817744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465817744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465817744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,768 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/e052fb7f9f8642fcaaa4dda0eda31ba1 2024-12-06T06:15:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/c97467280016466fa00c2be9b53e09b2 is 50, key is test_row_0/B:col10/1733465756288/Put/seqid=0 2024-12-06T06:15:57,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742330_1506 (size=9657) 2024-12-06T06:15:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T06:15:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465817903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465817943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465817964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465817964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465817964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,178 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/c97467280016466fa00c2be9b53e09b2 2024-12-06T06:15:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/7e79f7fbc7a64c91864a3d73b00db2e6 is 50, key is test_row_0/C:col10/1733465756288/Put/seqid=0 2024-12-06T06:15:58,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742331_1507 (size=9657) 2024-12-06T06:15:58,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465818224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465818263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465818284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465818284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465818284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T06:15:58,588 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/7e79f7fbc7a64c91864a3d73b00db2e6 2024-12-06T06:15:58,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/e052fb7f9f8642fcaaa4dda0eda31ba1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/e052fb7f9f8642fcaaa4dda0eda31ba1 2024-12-06T06:15:58,594 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/e052fb7f9f8642fcaaa4dda0eda31ba1, entries=100, sequenceid=125, filesize=9.4 K 2024-12-06T06:15:58,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/c97467280016466fa00c2be9b53e09b2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/c97467280016466fa00c2be9b53e09b2 2024-12-06T06:15:58,598 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/c97467280016466fa00c2be9b53e09b2, entries=100, sequenceid=125, filesize=9.4 K 2024-12-06T06:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/7e79f7fbc7a64c91864a3d73b00db2e6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7e79f7fbc7a64c91864a3d73b00db2e6 2024-12-06T06:15:58,602 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7e79f7fbc7a64c91864a3d73b00db2e6, entries=100, sequenceid=125, filesize=9.4 K 2024-12-06T06:15:58,602 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 6da2c4ed30e0a36b951d14c383b1d513 in 1243ms, sequenceid=125, compaction requested=false 2024-12-06T06:15:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:15:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-06T06:15:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-06T06:15:58,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-06T06:15:58,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-12-06T06:15:58,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4010 sec 2024-12-06T06:15:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:15:58,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-12-06T06:15:58,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:15:58,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:58,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:15:58,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:58,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:15:58,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:15:58,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/1b90b52052e74683b299d068fc1ad8af is 50, key is test_row_0/A:col10/1733465757609/Put/seqid=0 2024-12-06T06:15:58,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742332_1508 (size=14541) 2024-12-06T06:15:58,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465818804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465818804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465818804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465818806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465818806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465818923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465818923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465819144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465819144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/1b90b52052e74683b299d068fc1ad8af 2024-12-06T06:15:59,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/e812353ecf5d40e2bce78f4ccb8693bd is 50, key is test_row_0/B:col10/1733465757609/Put/seqid=0 2024-12-06T06:15:59,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742333_1509 (size=12151) 2024-12-06T06:15:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T06:15:59,354 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:15:59,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:15:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-06T06:15:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T06:15:59,356 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:15:59,357 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:15:59,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T06:15:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465819464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465819464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T06:15:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:59,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/e812353ecf5d40e2bce78f4ccb8693bd 2024-12-06T06:15:59,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/5e625ad0fbfa4943898d021acc67ab1d is 50, key is test_row_0/C:col10/1733465757609/Put/seqid=0 2024-12-06T06:15:59,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742334_1510 (size=12151) 2024-12-06T06:15:59,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T06:15:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:59,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T06:15:59,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T06:15:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:59,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465819834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465819834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465819834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,844 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:59,844 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,844 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:15:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T06:15:59,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:59,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:15:59,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:15:59,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:15:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T06:15:59,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/5e625ad0fbfa4943898d021acc67ab1d 2024-12-06T06:15:59,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/1b90b52052e74683b299d068fc1ad8af as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/1b90b52052e74683b299d068fc1ad8af 2024-12-06T06:15:59,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/1b90b52052e74683b299d068fc1ad8af, entries=200, sequenceid=198, filesize=14.2 K 2024-12-06T06:15:59,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/e812353ecf5d40e2bce78f4ccb8693bd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e812353ecf5d40e2bce78f4ccb8693bd 2024-12-06T06:15:59,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e812353ecf5d40e2bce78f4ccb8693bd, entries=150, sequenceid=198, filesize=11.9 K 2024-12-06T06:15:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465819983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:15:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465819983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:15:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:15:59,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/5e625ad0fbfa4943898d021acc67ab1d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/5e625ad0fbfa4943898d021acc67ab1d 2024-12-06T06:15:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:15:59,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/5e625ad0fbfa4943898d021acc67ab1d, entries=150, sequenceid=198, filesize=11.9 K 2024-12-06T06:16:00,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~434.34 KB/444768, currentSize=33.54 KB/34350 for 6da2c4ed30e0a36b951d14c383b1d513 in 1246ms, sequenceid=198, compaction requested=true 2024-12-06T06:16:00,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:00,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:00,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:00,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:00,000 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:00,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:00,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:00,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:00,000 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:00,001 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36337 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:00,002 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/A is initiating minor compaction (all files) 2024-12-06T06:16:00,002 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:00,002 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/A in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:00,002 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/B is initiating minor compaction (all files) 2024-12-06T06:16:00,002 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c1b5669aa93f41c7b721db84da2e4805, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/e052fb7f9f8642fcaaa4dda0eda31ba1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/1b90b52052e74683b299d068fc1ad8af] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=35.5 K 2024-12-06T06:16:00,002 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/B in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:00,002 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/24e247e3a11b41bebb916075857343cf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/c97467280016466fa00c2be9b53e09b2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e812353ecf5d40e2bce78f4ccb8693bd] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=33.2 K 2024-12-06T06:16:00,002 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b5669aa93f41c7b721db84da2e4805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465755605 2024-12-06T06:16:00,002 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e247e3a11b41bebb916075857343cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465755605 2024-12-06T06:16:00,002 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e052fb7f9f8642fcaaa4dda0eda31ba1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733465756288 2024-12-06T06:16:00,003 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b90b52052e74683b299d068fc1ad8af, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733465757609 2024-12-06T06:16:00,003 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting c97467280016466fa00c2be9b53e09b2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733465756288 2024-12-06T06:16:00,003 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e812353ecf5d40e2bce78f4ccb8693bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733465757609 2024-12-06T06:16:00,010 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:00,010 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#A#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:00,011 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/d2eed472b9bc472298d6aa5949553b98 is 50, key is test_row_0/B:col10/1733465757609/Put/seqid=0 2024-12-06T06:16:00,011 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/8ad5d490a1524bed99d104d51cf8b758 is 50, key is test_row_0/A:col10/1733465757609/Put/seqid=0 2024-12-06T06:16:00,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742336_1512 (size=12391) 2024-12-06T06:16:00,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742335_1511 (size=12391) 2024-12-06T06:16:00,032 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/d2eed472b9bc472298d6aa5949553b98 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/d2eed472b9bc472298d6aa5949553b98 2024-12-06T06:16:00,038 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/B of 6da2c4ed30e0a36b951d14c383b1d513 into d2eed472b9bc472298d6aa5949553b98(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:00,038 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:00,038 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/B, priority=13, startTime=1733465760000; duration=0sec 2024-12-06T06:16:00,038 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:00,038 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:B 2024-12-06T06:16:00,039 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:00,039 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:00,039 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/C is initiating minor compaction (all files) 2024-12-06T06:16:00,039 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/C in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:00,040 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/c62f735c2fc840d4951d8fbe5eae3737, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7e79f7fbc7a64c91864a3d73b00db2e6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/5e625ad0fbfa4943898d021acc67ab1d] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=33.2 K 2024-12-06T06:16:00,040 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting c62f735c2fc840d4951d8fbe5eae3737, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733465755605 2024-12-06T06:16:00,040 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e79f7fbc7a64c91864a3d73b00db2e6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733465756288 2024-12-06T06:16:00,041 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e625ad0fbfa4943898d021acc67ab1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733465757609 2024-12-06T06:16:00,050 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:00,050 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/215ee22d87fe4945b27a0e5650bcd095 is 50, key is test_row_0/C:col10/1733465757609/Put/seqid=0 2024-12-06T06:16:00,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742337_1513 (size=12391) 2024-12-06T06:16:00,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T06:16:00,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:00,121 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T06:16:00,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:00,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:00,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:00,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:00,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:00,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:00,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/5c1d5e900e17442aa1f8c7c8bd44f6b1 is 50, key is test_row_1/A:col10/1733465758793/Put/seqid=0 2024-12-06T06:16:00,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742338_1514 (size=9757) 2024-12-06T06:16:00,432 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/8ad5d490a1524bed99d104d51cf8b758 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/8ad5d490a1524bed99d104d51cf8b758 2024-12-06T06:16:00,436 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/A of 6da2c4ed30e0a36b951d14c383b1d513 into 8ad5d490a1524bed99d104d51cf8b758(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:00,436 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:00,436 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/A, priority=13, startTime=1733465760000; duration=0sec 2024-12-06T06:16:00,436 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:00,436 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:A 2024-12-06T06:16:00,459 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/215ee22d87fe4945b27a0e5650bcd095 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/215ee22d87fe4945b27a0e5650bcd095 2024-12-06T06:16:00,463 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/C of 6da2c4ed30e0a36b951d14c383b1d513 into 215ee22d87fe4945b27a0e5650bcd095(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:00,463 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:00,463 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/C, priority=13, startTime=1733465760000; duration=0sec 2024-12-06T06:16:00,463 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:00,463 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:C 2024-12-06T06:16:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T06:16:00,529 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/5c1d5e900e17442aa1f8c7c8bd44f6b1 2024-12-06T06:16:00,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/277f79dea99d43ec8811d2288fb9716d is 50, key is test_row_1/B:col10/1733465758793/Put/seqid=0 2024-12-06T06:16:00,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742339_1515 (size=9757) 2024-12-06T06:16:00,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T06:16:00,713 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-06T06:16:00,941 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/277f79dea99d43ec8811d2288fb9716d 2024-12-06T06:16:00,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/b914d7ae8197447889e2cb951baac0f4 is 50, key is test_row_1/C:col10/1733465758793/Put/seqid=0 2024-12-06T06:16:00,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742340_1516 (size=9757) 2024-12-06T06:16:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:01,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:01,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465761031, max timestamp=1733465761068Num uniques 859; 2024-12-06T06:16:01,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465761031, max timestamp=1733465761068Num uniques 859; 2024-12-06T06:16:01,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:01,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:01,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465761031, max timestamp=1733465761068Num uniques 859; 2024-12-06T06:16:01,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465821140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733465761106, max timestamp=1733465761148Num uniques 1068; 2024-12-06T06:16:01,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733465761106, max timestamp=1733465761148Num uniques 1068; 2024-12-06T06:16:01,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T06:16:01,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T06:16:01,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733465761106, max timestamp=1733465761148Num uniques 1068; 2024-12-06T06:16:01,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T06:16:01,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733465761232, max timestamp=1733465761279Num uniques 1027; 2024-12-06T06:16:01,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733465761232, max timestamp=1733465761279Num uniques 1027; 2024-12-06T06:16:01,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-06T06:16:01,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-06T06:16:01,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733465761232, max timestamp=1733465761279Num uniques 1027; 2024-12-06T06:16:01,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-06T06:16:01,354 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/b914d7ae8197447889e2cb951baac0f4 2024-12-06T06:16:01,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/5c1d5e900e17442aa1f8c7c8bd44f6b1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/5c1d5e900e17442aa1f8c7c8bd44f6b1 2024-12-06T06:16:01,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733465761320, max timestamp=1733465761360Num uniques 1036; 2024-12-06T06:16:01,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-06T06:16:01,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733465761320, max timestamp=1733465761360Num uniques 1036; 2024-12-06T06:16:01,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-06T06:16:01,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733465761320, max timestamp=1733465761360Num uniques 1036; 2024-12-06T06:16:01,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-06T06:16:01,366 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/5c1d5e900e17442aa1f8c7c8bd44f6b1, entries=100, sequenceid=207, filesize=9.5 K 2024-12-06T06:16:01,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/277f79dea99d43ec8811d2288fb9716d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/277f79dea99d43ec8811d2288fb9716d 2024-12-06T06:16:01,370 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/277f79dea99d43ec8811d2288fb9716d, entries=100, sequenceid=207, filesize=9.5 K 2024-12-06T06:16:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/b914d7ae8197447889e2cb951baac0f4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b914d7ae8197447889e2cb951baac0f4 2024-12-06T06:16:01,374 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b914d7ae8197447889e2cb951baac0f4, entries=100, sequenceid=207, filesize=9.5 K 2024-12-06T06:16:01,375 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=40.25 KB/41220 for 6da2c4ed30e0a36b951d14c383b1d513 in 1254ms, sequenceid=207, compaction requested=false 2024-12-06T06:16:01,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:01,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:01,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-06T06:16:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-06T06:16:01,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-06T06:16:01,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-12-06T06:16:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:01,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=53.67 KB heapSize=138.70 KB 2024-12-06T06:16:01,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:01,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:01,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:01,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:01,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:01,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:01,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/763d6eb10af94de8b7f77c79aebd83b2 is 50, key is test_row_0/A:col10/1733465761358/Put/seqid=0 2024-12-06T06:16:01,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0250 sec 2024-12-06T06:16:01,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742341_1517 (size=14741) 2024-12-06T06:16:01,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465761427, max timestamp=1733465761466Num uniques 859; 2024-12-06T06:16:01,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465761427, max timestamp=1733465761466Num uniques 859; 2024-12-06T06:16:01,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733465761427, max timestamp=1733465761466Num uniques 859; 2024-12-06T06:16:01,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:16:01,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:16:01,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-06T06:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T06:16:01,514 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:01,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-06T06:16:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T06:16:01,516 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:01,517 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:01,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:01,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465821554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465821560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T06:16:01,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:01,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:01,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:01,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:01,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:01,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:01,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465821673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465821683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/763d6eb10af94de8b7f77c79aebd83b2 2024-12-06T06:16:01,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/196243bcd2114aecb466fbdfd765e3e8 is 50, key is test_row_0/B:col10/1733465761358/Put/seqid=0 2024-12-06T06:16:01,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742342_1518 (size=12301) 2024-12-06T06:16:01,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:01,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:01,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:01,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:01,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:01,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T06:16:01,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465821854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,866 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465821863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465821863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,872 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,873 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465821894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465821894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:01,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:01,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:01,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:01,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:01,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:01,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T06:16:02,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/196243bcd2114aecb466fbdfd765e3e8 2024-12-06T06:16:02,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/b4d192df39bf4020ba6db9fbc6670a96 is 50, key is test_row_0/C:col10/1733465761358/Put/seqid=0 2024-12-06T06:16:02,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742343_1519 (size=12301) 2024-12-06T06:16:02,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465822213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:02,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:02,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465822213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:02,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:02,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/b4d192df39bf4020ba6db9fbc6670a96 2024-12-06T06:16:02,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/763d6eb10af94de8b7f77c79aebd83b2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/763d6eb10af94de8b7f77c79aebd83b2 2024-12-06T06:16:02,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/763d6eb10af94de8b7f77c79aebd83b2, entries=200, sequenceid=284, filesize=14.4 K 2024-12-06T06:16:02,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/196243bcd2114aecb466fbdfd765e3e8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/196243bcd2114aecb466fbdfd765e3e8 2024-12-06T06:16:02,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/196243bcd2114aecb466fbdfd765e3e8, entries=150, sequenceid=284, filesize=12.0 K 2024-12-06T06:16:02,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/b4d192df39bf4020ba6db9fbc6670a96 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b4d192df39bf4020ba6db9fbc6670a96 2024-12-06T06:16:02,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b4d192df39bf4020ba6db9fbc6670a96, entries=150, sequenceid=284, filesize=12.0 K 2024-12-06T06:16:02,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~138.66 KB/141984, currentSize=134.18 KB/137400 for 6da2c4ed30e0a36b951d14c383b1d513 in 1256ms, sequenceid=284, compaction requested=true 2024-12-06T06:16:02,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:02,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:02,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:02,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:02,634 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:02,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:02,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:02,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:16:02,635 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:02,636 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:02,636 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/B is initiating minor compaction (all files) 2024-12-06T06:16:02,636 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/B in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,636 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/d2eed472b9bc472298d6aa5949553b98, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/277f79dea99d43ec8811d2288fb9716d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/196243bcd2114aecb466fbdfd765e3e8] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=33.6 K 2024-12-06T06:16:02,636 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:02,636 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/A is initiating minor compaction (all files) 2024-12-06T06:16:02,636 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/A in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,636 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/8ad5d490a1524bed99d104d51cf8b758, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/5c1d5e900e17442aa1f8c7c8bd44f6b1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/763d6eb10af94de8b7f77c79aebd83b2] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=36.0 K 2024-12-06T06:16:02,638 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d2eed472b9bc472298d6aa5949553b98, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733465757609 2024-12-06T06:16:02,638 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad5d490a1524bed99d104d51cf8b758, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733465757609 2024-12-06T06:16:02,639 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1d5e900e17442aa1f8c7c8bd44f6b1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465758781 2024-12-06T06:16:02,639 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 277f79dea99d43ec8811d2288fb9716d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465758781 2024-12-06T06:16:02,639 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763d6eb10af94de8b7f77c79aebd83b2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733465761320 2024-12-06T06:16:02,639 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 196243bcd2114aecb466fbdfd765e3e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733465761320 2024-12-06T06:16:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T06:16:02,655 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#A#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:02,655 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/da193b332d0a47d5b0bf0b0b28aa34eb is 50, key is test_row_0/A:col10/1733465761358/Put/seqid=0 2024-12-06T06:16:02,659 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:02,659 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/e6745b2e143e47fabb7b8a63a66589a4 is 50, key is test_row_0/B:col10/1733465761358/Put/seqid=0 2024-12-06T06:16:02,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742344_1520 (size=12643) 2024-12-06T06:16:02,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742345_1521 (size=12643) 2024-12-06T06:16:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:02,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-06T06:16:02,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:16:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:16:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-06T06:16:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/f680783d375e41cfb79c9f3b42752e0b is 50, key is test_row_0/A:col10/1733465761552/Put/seqid=0 2024-12-06T06:16:02,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742346_1522 (size=14741) 2024-12-06T06:16:02,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/f680783d375e41cfb79c9f3b42752e0b 2024-12-06T06:16:02,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/4286a1e86d2d405c97116d133ebaf1d6 is 50, key is test_row_0/B:col10/1733465761552/Put/seqid=0 2024-12-06T06:16:02,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742347_1523 (size=12301) 2024-12-06T06:16:02,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465822770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:02,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465822781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:02,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:02,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465822893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:02,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465822903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:03,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,089 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/da193b332d0a47d5b0bf0b0b28aa34eb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/da193b332d0a47d5b0bf0b0b28aa34eb 2024-12-06T06:16:03,093 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/A of 6da2c4ed30e0a36b951d14c383b1d513 into da193b332d0a47d5b0bf0b0b28aa34eb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:03,093 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:03,093 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/A, priority=13, startTime=1733465762634; duration=0sec 2024-12-06T06:16:03,093 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:03,093 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:A 2024-12-06T06:16:03,093 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:03,094 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:03,094 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/C is initiating minor compaction (all files) 2024-12-06T06:16:03,094 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/C in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,094 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/215ee22d87fe4945b27a0e5650bcd095, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b914d7ae8197447889e2cb951baac0f4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b4d192df39bf4020ba6db9fbc6670a96] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=33.6 K 2024-12-06T06:16:03,095 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 215ee22d87fe4945b27a0e5650bcd095, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733465757609 2024-12-06T06:16:03,095 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b914d7ae8197447889e2cb951baac0f4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465758781 2024-12-06T06:16:03,095 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4d192df39bf4020ba6db9fbc6670a96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733465761320 2024-12-06T06:16:03,100 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:03,101 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/f71e70b293804899b36724b9679da993 is 50, key is test_row_0/C:col10/1733465761358/Put/seqid=0 2024-12-06T06:16:03,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742348_1524 (size=12643) 2024-12-06T06:16:03,107 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/e6745b2e143e47fabb7b8a63a66589a4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e6745b2e143e47fabb7b8a63a66589a4 2024-12-06T06:16:03,111 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/f71e70b293804899b36724b9679da993 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f71e70b293804899b36724b9679da993 2024-12-06T06:16:03,111 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/B of 6da2c4ed30e0a36b951d14c383b1d513 into e6745b2e143e47fabb7b8a63a66589a4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:03,111 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:03,112 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/B, priority=13, startTime=1733465762634; duration=0sec 2024-12-06T06:16:03,112 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:03,112 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:B 2024-12-06T06:16:03,115 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/C of 6da2c4ed30e0a36b951d14c383b1d513 into f71e70b293804899b36724b9679da993(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:03,115 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:03,115 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/C, priority=13, startTime=1733465762634; duration=0sec 2024-12-06T06:16:03,115 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:03,115 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:C 2024-12-06T06:16:03,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465823114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:03,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:03,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:03,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:03,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465823123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:03,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/4286a1e86d2d405c97116d133ebaf1d6 2024-12-06T06:16:03,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/137e90200a634c4fb581fc17434451ad is 50, key is test_row_0/C:col10/1733465761552/Put/seqid=0 2024-12-06T06:16:03,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742349_1525 (size=12301) 2024-12-06T06:16:03,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:03,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:03,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465823424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465823444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:03,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:03,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:03,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:03,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/137e90200a634c4fb581fc17434451ad 2024-12-06T06:16:03,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/f680783d375e41cfb79c9f3b42752e0b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f680783d375e41cfb79c9f3b42752e0b 2024-12-06T06:16:03,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f680783d375e41cfb79c9f3b42752e0b, entries=200, sequenceid=322, filesize=14.4 K 2024-12-06T06:16:03,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/4286a1e86d2d405c97116d133ebaf1d6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/4286a1e86d2d405c97116d133ebaf1d6 2024-12-06T06:16:03,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/4286a1e86d2d405c97116d133ebaf1d6, entries=150, sequenceid=322, filesize=12.0 K 2024-12-06T06:16:03,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/137e90200a634c4fb581fc17434451ad as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/137e90200a634c4fb581fc17434451ad 2024-12-06T06:16:03,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/137e90200a634c4fb581fc17434451ad, entries=150, sequenceid=322, filesize=12.0 K 2024-12-06T06:16:03,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 6da2c4ed30e0a36b951d14c383b1d513 in 858ms, sequenceid=322, compaction requested=false 2024-12-06T06:16:03,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:03,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-06T06:16:03,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:03,662 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-06T06:16:03,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:03,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:03,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:03,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:03,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:03,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T06:16:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/c4416e0805534dc2b48ac67920ad644a is 50, key is test_row_0/A:col10/1733465762768/Put/seqid=0 2024-12-06T06:16:03,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742350_1526 (size=12301) 2024-12-06T06:16:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:03,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465764009, max timestamp=1733465764040Num uniques 859; 2024-12-06T06:16:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:04,071 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/c4416e0805534dc2b48ac67920ad644a 2024-12-06T06:16:04,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/cb3b1ca7423642efbad7fc13d53b1262 is 50, key is test_row_0/B:col10/1733465762768/Put/seqid=0 2024-12-06T06:16:04,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742351_1527 (size=12301) 2024-12-06T06:16:04,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465824078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:04,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465824090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:04,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465824194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:04,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465824203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:04,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465824414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:04,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465824424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:04,481 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/cb3b1ca7423642efbad7fc13d53b1262 2024-12-06T06:16:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/ea3816c887a043d7abb32ee1937af5e0 is 50, key is test_row_0/C:col10/1733465762768/Put/seqid=0 2024-12-06T06:16:04,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742352_1528 (size=12301) 2024-12-06T06:16:04,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465824733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:04,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:04,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465824743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:04,891 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/ea3816c887a043d7abb32ee1937af5e0 2024-12-06T06:16:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/c4416e0805534dc2b48ac67920ad644a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c4416e0805534dc2b48ac67920ad644a 2024-12-06T06:16:04,899 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c4416e0805534dc2b48ac67920ad644a, entries=150, sequenceid=334, filesize=12.0 K 2024-12-06T06:16:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/cb3b1ca7423642efbad7fc13d53b1262 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/cb3b1ca7423642efbad7fc13d53b1262 2024-12-06T06:16:04,902 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/cb3b1ca7423642efbad7fc13d53b1262, entries=150, sequenceid=334, filesize=12.0 K 2024-12-06T06:16:04,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/ea3816c887a043d7abb32ee1937af5e0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ea3816c887a043d7abb32ee1937af5e0 2024-12-06T06:16:04,906 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ea3816c887a043d7abb32ee1937af5e0, entries=150, sequenceid=334, filesize=12.0 K 2024-12-06T06:16:04,906 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for 6da2c4ed30e0a36b951d14c383b1d513 in 1244ms, sequenceid=334, compaction requested=true 2024-12-06T06:16:04,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:04,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:04,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-06T06:16:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-06T06:16:04,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-06T06:16:04,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3910 sec 2024-12-06T06:16:04,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.3960 sec 2024-12-06T06:16:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:05,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=145.36 KB heapSize=426.56 KB 2024-12-06T06:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:05,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/968bb57f274e4c978b17e02c6f376bd4 is 50, key is test_row_0/A:col10/1733465764076/Put/seqid=0 2024-12-06T06:16:05,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742353_1529 (size=14741) 2024-12-06T06:16:05,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465825289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465825292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465825403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465825413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465825614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465825633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/968bb57f274e4c978b17e02c6f376bd4 2024-12-06T06:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T06:16:05,674 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:05,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-06T06:16:05,676 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T06:16:05,677 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:05,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:05,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/72761a0ed92b4447adee5c3ac6ee68e5 is 50, key is test_row_0/B:col10/1733465764076/Put/seqid=0 2024-12-06T06:16:05,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742354_1530 (size=12301) 2024-12-06T06:16:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T06:16:05,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T06:16:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:05,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:05,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:05,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54286 deadline: 1733465825893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54296 deadline: 1733465825893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1733465825893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,908 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:05,909 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,909 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in 6da2c4ed30e0a36b951d14c383b1d513 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465825934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465825953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:05,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T06:16:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:05,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T06:16:06,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/72761a0ed92b4447adee5c3ac6ee68e5 2024-12-06T06:16:06,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/27da05b979d34a17b52ecadc707bbe2d is 50, key is test_row_0/C:col10/1733465764076/Put/seqid=0 2024-12-06T06:16:06,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742355_1531 (size=12301) 2024-12-06T06:16:06,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T06:16:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T06:16:06,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:06,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T06:16:06,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T06:16:06,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:06,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:06,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465826454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:06,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465826473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:06,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/27da05b979d34a17b52ecadc707bbe2d 2024-12-06T06:16:06,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/968bb57f274e4c978b17e02c6f376bd4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/968bb57f274e4c978b17e02c6f376bd4 2024-12-06T06:16:06,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/968bb57f274e4c978b17e02c6f376bd4, entries=200, sequenceid=363, filesize=14.4 K 2024-12-06T06:16:06,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/72761a0ed92b4447adee5c3ac6ee68e5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/72761a0ed92b4447adee5c3ac6ee68e5 2024-12-06T06:16:06,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/72761a0ed92b4447adee5c3ac6ee68e5, entries=150, sequenceid=363, filesize=12.0 K 2024-12-06T06:16:06,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/27da05b979d34a17b52ecadc707bbe2d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/27da05b979d34a17b52ecadc707bbe2d 2024-12-06T06:16:06,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/27da05b979d34a17b52ecadc707bbe2d, entries=150, sequenceid=363, filesize=12.0 K 2024-12-06T06:16:06,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~426.52 KB/436752, currentSize=33.54 KB/34350 for 6da2c4ed30e0a36b951d14c383b1d513 in 1244ms, sequenceid=363, compaction requested=true 2024-12-06T06:16:06,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:06,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:06,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:06,508 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:06,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:06,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:06,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:06,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:06,508 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:06,509 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:06,509 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54426 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:06,509 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/A is initiating minor compaction (all files) 2024-12-06T06:16:06,509 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/B is initiating minor compaction (all files) 2024-12-06T06:16:06,509 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/A in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,509 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/B in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,509 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/da193b332d0a47d5b0bf0b0b28aa34eb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f680783d375e41cfb79c9f3b42752e0b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c4416e0805534dc2b48ac67920ad644a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/968bb57f274e4c978b17e02c6f376bd4] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=53.2 K 2024-12-06T06:16:06,509 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e6745b2e143e47fabb7b8a63a66589a4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/4286a1e86d2d405c97116d133ebaf1d6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/cb3b1ca7423642efbad7fc13d53b1262, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/72761a0ed92b4447adee5c3ac6ee68e5] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=48.4 K 2024-12-06T06:16:06,509 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting da193b332d0a47d5b0bf0b0b28aa34eb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733465761320 2024-12-06T06:16:06,509 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e6745b2e143e47fabb7b8a63a66589a4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733465761320 2024-12-06T06:16:06,510 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f680783d375e41cfb79c9f3b42752e0b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733465761535 2024-12-06T06:16:06,510 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4286a1e86d2d405c97116d133ebaf1d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733465761552 2024-12-06T06:16:06,511 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4416e0805534dc2b48ac67920ad644a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733465762767 2024-12-06T06:16:06,511 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3b1ca7423642efbad7fc13d53b1262, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733465762767 2024-12-06T06:16:06,511 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 968bb57f274e4c978b17e02c6f376bd4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733465764071 2024-12-06T06:16:06,511 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 72761a0ed92b4447adee5c3ac6ee68e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733465764076 2024-12-06T06:16:06,525 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#B#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:06,525 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/93e5feca64f64d088e494fb97949c095 is 50, key is test_row_0/B:col10/1733465764076/Put/seqid=0 2024-12-06T06:16:06,526 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#A#compaction#456 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:06,526 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/d4aa9c3180514bcbb6910e8fe818d70c is 50, key is test_row_0/A:col10/1733465764076/Put/seqid=0 2024-12-06T06:16:06,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742356_1532 (size=12779) 2024-12-06T06:16:06,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742357_1533 (size=12779) 2024-12-06T06:16:06,535 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/93e5feca64f64d088e494fb97949c095 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/93e5feca64f64d088e494fb97949c095 2024-12-06T06:16:06,540 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/B of 6da2c4ed30e0a36b951d14c383b1d513 into 93e5feca64f64d088e494fb97949c095(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:06,540 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:06,540 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/B, priority=12, startTime=1733465766508; duration=0sec 2024-12-06T06:16:06,540 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:06,540 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:B 2024-12-06T06:16:06,540 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:06,541 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:06,541 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/C is initiating minor compaction (all files) 2024-12-06T06:16:06,541 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/C in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,541 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f71e70b293804899b36724b9679da993, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/137e90200a634c4fb581fc17434451ad, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ea3816c887a043d7abb32ee1937af5e0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/27da05b979d34a17b52ecadc707bbe2d] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=48.4 K 2024-12-06T06:16:06,541 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting f71e70b293804899b36724b9679da993, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733465761320 2024-12-06T06:16:06,541 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 137e90200a634c4fb581fc17434451ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733465761552 2024-12-06T06:16:06,542 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3816c887a043d7abb32ee1937af5e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733465762767 2024-12-06T06:16:06,542 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 27da05b979d34a17b52ecadc707bbe2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733465764076 2024-12-06T06:16:06,549 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#C#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:06,549 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/3513fba61ca44f22823e8d9b5c8638b5 is 50, key is test_row_0/C:col10/1733465764076/Put/seqid=0 2024-12-06T06:16:06,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742358_1534 (size=12779) 2024-12-06T06:16:06,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-06T06:16:06,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:06,594 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T06:16:06,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:06,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:06,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:06,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:06,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:06,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/fca8f889bfda4025a3b8f1710594c3dc is 50, key is test_row_0/A:col10/1733465765273/Put/seqid=0 2024-12-06T06:16:06,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742359_1535 (size=12301) 2024-12-06T06:16:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T06:16:06,935 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/d4aa9c3180514bcbb6910e8fe818d70c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/d4aa9c3180514bcbb6910e8fe818d70c 2024-12-06T06:16:06,940 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/A of 6da2c4ed30e0a36b951d14c383b1d513 into d4aa9c3180514bcbb6910e8fe818d70c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:06,940 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:06,940 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/A, priority=12, startTime=1733465766508; duration=0sec 2024-12-06T06:16:06,940 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:06,940 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:A 2024-12-06T06:16:06,960 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/3513fba61ca44f22823e8d9b5c8638b5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/3513fba61ca44f22823e8d9b5c8638b5 2024-12-06T06:16:06,965 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/C of 6da2c4ed30e0a36b951d14c383b1d513 into 3513fba61ca44f22823e8d9b5c8638b5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:06,965 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:06,965 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/C, priority=12, startTime=1733465766508; duration=0sec 2024-12-06T06:16:06,965 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:06,965 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:C 2024-12-06T06:16:07,002 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/fca8f889bfda4025a3b8f1710594c3dc 2024-12-06T06:16:07,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/79c5aacec50a43efb3df454009513ee3 is 50, key is test_row_0/B:col10/1733465765273/Put/seqid=0 2024-12-06T06:16:07,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742360_1536 (size=12301) 2024-12-06T06:16:07,414 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/79c5aacec50a43efb3df454009513ee3 2024-12-06T06:16:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/21a01204d93b4e50b00095a0e5af4562 is 50, key is test_row_0/C:col10/1733465765273/Put/seqid=0 2024-12-06T06:16:07,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742361_1537 (size=12301) 2024-12-06T06:16:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:07,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:07,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465767526, max timestamp=1733465767558Num uniques 859; 2024-12-06T06:16:07,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:07,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465767526, max timestamp=1733465767558Num uniques 859; 2024-12-06T06:16:07,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:07,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465767526, max timestamp=1733465767558Num uniques 859; 2024-12-06T06:16:07,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:07,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733465767606, max timestamp=1733465767647Num uniques 1068; 2024-12-06T06:16:07,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T06:16:07,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733465767606, max timestamp=1733465767647Num uniques 1068; 2024-12-06T06:16:07,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T06:16:07,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733465767606, max timestamp=1733465767647Num uniques 1068; 2024-12-06T06:16:07,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-06T06:16:07,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733465767701, max timestamp=1733465767761Num uniques 1027; 2024-12-06T06:16:07,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733465767701, max timestamp=1733465767761Num uniques 1027; 2024-12-06T06:16:07,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-06T06:16:07,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-06T06:16:07,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733465767701, max timestamp=1733465767761Num uniques 1027; 2024-12-06T06:16:07,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-06T06:16:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T06:16:07,824 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/21a01204d93b4e50b00095a0e5af4562 2024-12-06T06:16:07,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/fca8f889bfda4025a3b8f1710594c3dc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/fca8f889bfda4025a3b8f1710594c3dc 2024-12-06T06:16:07,831 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/fca8f889bfda4025a3b8f1710594c3dc, entries=150, sequenceid=372, filesize=12.0 K 2024-12-06T06:16:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/79c5aacec50a43efb3df454009513ee3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/79c5aacec50a43efb3df454009513ee3 2024-12-06T06:16:07,835 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/79c5aacec50a43efb3df454009513ee3, entries=150, sequenceid=372, filesize=12.0 K 2024-12-06T06:16:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/21a01204d93b4e50b00095a0e5af4562 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/21a01204d93b4e50b00095a0e5af4562 2024-12-06T06:16:07,838 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/21a01204d93b4e50b00095a0e5af4562, entries=150, sequenceid=372, filesize=12.0 K 2024-12-06T06:16:07,838 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=120.76 KB/123660 for 6da2c4ed30e0a36b951d14c383b1d513 in 1244ms, sequenceid=372, compaction requested=false 2024-12-06T06:16:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-06T06:16:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-06T06:16:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:07,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=127.47 KB heapSize=358.69 KB 2024-12-06T06:16:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:07,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-06T06:16:07,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-12-06T06:16:07,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.1680 sec 2024-12-06T06:16:07,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/58ef2b84c98f4ec190f743197297370e is 50, key is test_row_0/A:col10/1733465767827/Put/seqid=0 2024-12-06T06:16:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742362_1538 (size=14741) 2024-12-06T06:16:07,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465827880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:07,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465827887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465828003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:08,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465828003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:08,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465828214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:08,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465828215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:08,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:08,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/58ef2b84c98f4ec190f743197297370e 2024-12-06T06:16:08,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/db327e81d3824696b54aa0b69d32ccdb is 50, key is test_row_0/B:col10/1733465767827/Put/seqid=0 2024-12-06T06:16:08,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742363_1539 (size=12301) 2024-12-06T06:16:08,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465828524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:08,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465828533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:08,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/db327e81d3824696b54aa0b69d32ccdb 2024-12-06T06:16:08,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/f6b506ad080e4046b6a35550e98eb135 is 50, key is test_row_0/C:col10/1733465767827/Put/seqid=0 2024-12-06T06:16:08,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742364_1540 (size=12301) 2024-12-06T06:16:08,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/f6b506ad080e4046b6a35550e98eb135 2024-12-06T06:16:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/58ef2b84c98f4ec190f743197297370e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/58ef2b84c98f4ec190f743197297370e 2024-12-06T06:16:08,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/58ef2b84c98f4ec190f743197297370e, entries=200, sequenceid=443, filesize=14.4 K 2024-12-06T06:16:08,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/db327e81d3824696b54aa0b69d32ccdb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/db327e81d3824696b54aa0b69d32ccdb 2024-12-06T06:16:08,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/db327e81d3824696b54aa0b69d32ccdb, entries=150, sequenceid=443, filesize=12.0 K 2024-12-06T06:16:08,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/f6b506ad080e4046b6a35550e98eb135 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f6b506ad080e4046b6a35550e98eb135 2024-12-06T06:16:08,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f6b506ad080e4046b6a35550e98eb135, entries=150, sequenceid=443, filesize=12.0 K 2024-12-06T06:16:08,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~358.64 KB/367248, currentSize=53.67 KB/54960 for 6da2c4ed30e0a36b951d14c383b1d513 in 848ms, sequenceid=443, compaction requested=true 2024-12-06T06:16:08,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:08,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:08,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:08,690 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:08,690 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:08,690 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:08,690 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:08,690 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/A is initiating minor compaction (all files) 2024-12-06T06:16:08,690 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/A in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:08,690 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/B is initiating minor compaction (all files) 2024-12-06T06:16:08,690 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/B in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:08,690 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/d4aa9c3180514bcbb6910e8fe818d70c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/fca8f889bfda4025a3b8f1710594c3dc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/58ef2b84c98f4ec190f743197297370e] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=38.9 K 2024-12-06T06:16:08,691 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/93e5feca64f64d088e494fb97949c095, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/79c5aacec50a43efb3df454009513ee3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/db327e81d3824696b54aa0b69d32ccdb] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=36.5 K 2024-12-06T06:16:08,691 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4aa9c3180514bcbb6910e8fe818d70c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733465764076 2024-12-06T06:16:08,691 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e5feca64f64d088e494fb97949c095, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733465764076 2024-12-06T06:16:08,691 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca8f889bfda4025a3b8f1710594c3dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733465765273 2024-12-06T06:16:08,691 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c5aacec50a43efb3df454009513ee3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733465765273 2024-12-06T06:16:08,691 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58ef2b84c98f4ec190f743197297370e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733465767808 2024-12-06T06:16:08,691 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting db327e81d3824696b54aa0b69d32ccdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733465767824 2024-12-06T06:16:08,699 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:08,699 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/462293e4442f4da39d3176707e607ddf is 50, key is test_row_0/B:col10/1733465767827/Put/seqid=0 2024-12-06T06:16:08,700 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#A#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:08,700 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/737b41df4c9b495b8eaf26a218309be5 is 50, key is test_row_0/A:col10/1733465767827/Put/seqid=0 2024-12-06T06:16:08,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742365_1541 (size=12881) 2024-12-06T06:16:08,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742366_1542 (size=12881) 2024-12-06T06:16:08,718 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/737b41df4c9b495b8eaf26a218309be5 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/737b41df4c9b495b8eaf26a218309be5 2024-12-06T06:16:08,723 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/A of 6da2c4ed30e0a36b951d14c383b1d513 into 737b41df4c9b495b8eaf26a218309be5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:08,724 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:08,724 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/A, priority=13, startTime=1733465768689; duration=0sec 2024-12-06T06:16:08,724 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:08,724 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:A 2024-12-06T06:16:08,724 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:08,724 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:08,725 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/C is initiating minor compaction (all files) 2024-12-06T06:16:08,725 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/C in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:08,725 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/3513fba61ca44f22823e8d9b5c8638b5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/21a01204d93b4e50b00095a0e5af4562, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f6b506ad080e4046b6a35550e98eb135] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=36.5 K 2024-12-06T06:16:08,725 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3513fba61ca44f22823e8d9b5c8638b5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733465764076 2024-12-06T06:16:08,725 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a01204d93b4e50b00095a0e5af4562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733465765273 2024-12-06T06:16:08,725 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b506ad080e4046b6a35550e98eb135, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733465767824 2024-12-06T06:16:08,731 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#C#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:08,731 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/177035299f3047f7822fe71231b94bd7 is 50, key is test_row_0/C:col10/1733465767827/Put/seqid=0 2024-12-06T06:16:08,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742367_1543 (size=12881) 2024-12-06T06:16:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:09,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T06:16:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:09,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:09,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:09,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:09,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/cf1c41fa2907491d86edf9bbc17a5ab9 is 50, key is test_row_0/A:col10/1733465769039/Put/seqid=0 2024-12-06T06:16:09,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742368_1544 (size=14741) 2024-12-06T06:16:09,113 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/462293e4442f4da39d3176707e607ddf as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/462293e4442f4da39d3176707e607ddf 2024-12-06T06:16:09,117 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/B of 6da2c4ed30e0a36b951d14c383b1d513 into 462293e4442f4da39d3176707e607ddf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:09,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:09,117 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/B, priority=13, startTime=1733465768689; duration=0sec 2024-12-06T06:16:09,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:09,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:B 2024-12-06T06:16:09,138 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/177035299f3047f7822fe71231b94bd7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/177035299f3047f7822fe71231b94bd7 2024-12-06T06:16:09,141 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/C of 6da2c4ed30e0a36b951d14c383b1d513 into 177035299f3047f7822fe71231b94bd7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:09,141 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:09,141 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/C, priority=13, startTime=1733465768690; duration=0sec 2024-12-06T06:16:09,141 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:09,141 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:C 2024-12-06T06:16:09,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465829135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:09,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465829142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:09,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465829244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:09,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465829253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:09,290 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T06:16:09,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/cf1c41fa2907491d86edf9bbc17a5ab9 2024-12-06T06:16:09,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/88e0468e6bfc40fe9399b452563f0d1f is 50, key is test_row_0/B:col10/1733465769039/Put/seqid=0 2024-12-06T06:16:09,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742369_1545 (size=12301) 2024-12-06T06:16:09,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/88e0468e6bfc40fe9399b452563f0d1f 2024-12-06T06:16:09,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465829454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:09,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/015a0feb1348440a9b7a53ec7192daa0 is 50, key is test_row_0/C:col10/1733465769039/Put/seqid=0 2024-12-06T06:16:09,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742370_1546 (size=12301) 2024-12-06T06:16:09,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465829465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:09,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465829773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:09,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465829783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-06T06:16:09,834 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:09,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-06T06:16:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T06:16:09,837 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:09,837 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:09,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:09,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/015a0feb1348440a9b7a53ec7192daa0 2024-12-06T06:16:09,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/cf1c41fa2907491d86edf9bbc17a5ab9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cf1c41fa2907491d86edf9bbc17a5ab9 2024-12-06T06:16:09,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cf1c41fa2907491d86edf9bbc17a5ab9, entries=200, sequenceid=456, filesize=14.4 K 2024-12-06T06:16:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/88e0468e6bfc40fe9399b452563f0d1f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/88e0468e6bfc40fe9399b452563f0d1f 2024-12-06T06:16:09,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/88e0468e6bfc40fe9399b452563f0d1f, entries=150, sequenceid=456, filesize=12.0 K 2024-12-06T06:16:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/015a0feb1348440a9b7a53ec7192daa0 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/015a0feb1348440a9b7a53ec7192daa0 2024-12-06T06:16:09,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/015a0feb1348440a9b7a53ec7192daa0, entries=150, sequenceid=456, filesize=12.0 K 2024-12-06T06:16:09,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6da2c4ed30e0a36b951d14c383b1d513 in 850ms, sequenceid=456, compaction requested=false 2024-12-06T06:16:09,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T06:16:09,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-06T06:16:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:09,990 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:16:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:09,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/ba55eb06105c42bf9638ac5d5c350c8f is 50, key is test_row_0/A:col10/1733465769126/Put/seqid=0 2024-12-06T06:16:10,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742371_1547 (size=12301) 2024-12-06T06:16:10,001 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/ba55eb06105c42bf9638ac5d5c350c8f 2024-12-06T06:16:10,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/2c61d846c296425594a854001866daa8 is 50, key is test_row_0/B:col10/1733465769126/Put/seqid=0 2024-12-06T06:16:10,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742372_1548 (size=12301) 2024-12-06T06:16:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T06:16:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465830329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:10,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465830334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:10,413 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/2c61d846c296425594a854001866daa8 2024-12-06T06:16:10,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/9050045811f74ec1be5a574fdf44c98d is 50, key is test_row_0/C:col10/1733465769126/Put/seqid=0 2024-12-06T06:16:10,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742373_1549 (size=12301) 2024-12-06T06:16:10,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465830444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465830444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T06:16:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465830654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:10,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465830664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:10,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:10,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:10,824 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/9050045811f74ec1be5a574fdf44c98d 2024-12-06T06:16:10,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/ba55eb06105c42bf9638ac5d5c350c8f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/ba55eb06105c42bf9638ac5d5c350c8f 2024-12-06T06:16:10,832 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/ba55eb06105c42bf9638ac5d5c350c8f, entries=150, sequenceid=478, filesize=12.0 K 2024-12-06T06:16:10,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/2c61d846c296425594a854001866daa8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/2c61d846c296425594a854001866daa8 2024-12-06T06:16:10,836 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/2c61d846c296425594a854001866daa8, entries=150, sequenceid=478, filesize=12.0 K 2024-12-06T06:16:10,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/9050045811f74ec1be5a574fdf44c98d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9050045811f74ec1be5a574fdf44c98d 2024-12-06T06:16:10,842 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9050045811f74ec1be5a574fdf44c98d, entries=150, sequenceid=478, filesize=12.0 K 2024-12-06T06:16:10,843 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6da2c4ed30e0a36b951d14c383b1d513 in 853ms, sequenceid=478, compaction requested=true 2024-12-06T06:16:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-06T06:16:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-06T06:16:10,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-06T06:16:10,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0110 sec 2024-12-06T06:16:10,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.0160 sec 2024-12-06T06:16:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T06:16:10,974 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:10,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-06T06:16:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-06T06:16:10,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:10,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:10,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:10,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:16:10,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:10,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:10,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:10,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:10,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:10,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:10,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/73c624c8386d4b28b0b9c82c417c784c is 50, key is test_row_1/A:col10/1733465770979/Put/seqid=0 2024-12-06T06:16:10,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742374_1550 (size=12297) 2024-12-06T06:16:11,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465831051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:11,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465831065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-06T06:16:11,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:11,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:11,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465831173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:11,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465831173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:11,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:11,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:11,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-06T06:16:11,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/73c624c8386d4b28b0b9c82c417c784c 2024-12-06T06:16:11,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465831383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:11,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465831383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:11,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/0230c9e34c63474799fa8d2c11fe3bbd is 50, key is test_row_1/B:col10/1733465770979/Put/seqid=0 2024-12-06T06:16:11,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742375_1551 (size=9857) 2024-12-06T06:16:11,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:11,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:11,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:11,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:11,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-06T06:16:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465831704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:11,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465831704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:11,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/0230c9e34c63474799fa8d2c11fe3bbd 2024-12-06T06:16:11,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/ab4f8f160a2e46ff812a5784eeecf766 is 50, key is test_row_1/C:col10/1733465770979/Put/seqid=0 2024-12-06T06:16:11,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742376_1552 (size=9857) 2024-12-06T06:16:11,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:11,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:11,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:12,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:12,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:12,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:12,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:12,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:12,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-06T06:16:12,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:12,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:12,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. as already flushing 2024-12-06T06:16:12,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:12,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:12,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/ab4f8f160a2e46ff812a5784eeecf766 2024-12-06T06:16:12,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/73c624c8386d4b28b0b9c82c417c784c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/73c624c8386d4b28b0b9c82c417c784c 2024-12-06T06:16:12,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/73c624c8386d4b28b0b9c82c417c784c, entries=150, sequenceid=491, filesize=12.0 K 2024-12-06T06:16:12,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/0230c9e34c63474799fa8d2c11fe3bbd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0230c9e34c63474799fa8d2c11fe3bbd 2024-12-06T06:16:12,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54298 deadline: 1733465832224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1733465832224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 , the old value is region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., hostname=6336e8474e1c,39987,1733465620995, seqNum=2 because the exception is null or not the one we care about 2024-12-06T06:16:12,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0230c9e34c63474799fa8d2c11fe3bbd, entries=100, sequenceid=491, filesize=9.6 K 2024-12-06T06:16:12,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/ab4f8f160a2e46ff812a5784eeecf766 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ab4f8f160a2e46ff812a5784eeecf766 2024-12-06T06:16:12,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ab4f8f160a2e46ff812a5784eeecf766, entries=100, sequenceid=491, filesize=9.6 K 2024-12-06T06:16:12,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6da2c4ed30e0a36b951d14c383b1d513 in 1254ms, sequenceid=491, compaction requested=true 2024-12-06T06:16:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:12,234 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6da2c4ed30e0a36b951d14c383b1d513:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:12,234 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:12,235 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52220 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:12,235 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:12,235 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/A is initiating minor compaction (all files) 2024-12-06T06:16:12,235 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/B is initiating minor compaction (all files) 2024-12-06T06:16:12,235 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/A in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:12,235 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/B in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:12,235 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/737b41df4c9b495b8eaf26a218309be5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cf1c41fa2907491d86edf9bbc17a5ab9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/ba55eb06105c42bf9638ac5d5c350c8f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/73c624c8386d4b28b0b9c82c417c784c] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=51.0 K 2024-12-06T06:16:12,235 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/462293e4442f4da39d3176707e607ddf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/88e0468e6bfc40fe9399b452563f0d1f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/2c61d846c296425594a854001866daa8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0230c9e34c63474799fa8d2c11fe3bbd] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=46.2 K 2024-12-06T06:16:12,235 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 462293e4442f4da39d3176707e607ddf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733465767824 2024-12-06T06:16:12,235 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737b41df4c9b495b8eaf26a218309be5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733465767824 2024-12-06T06:16:12,236 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf1c41fa2907491d86edf9bbc17a5ab9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733465767867 2024-12-06T06:16:12,236 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 88e0468e6bfc40fe9399b452563f0d1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733465767867 2024-12-06T06:16:12,236 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c61d846c296425594a854001866daa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733465769108 2024-12-06T06:16:12,236 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba55eb06105c42bf9638ac5d5c350c8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733465769108 2024-12-06T06:16:12,236 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c624c8386d4b28b0b9c82c417c784c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733465770315 2024-12-06T06:16:12,236 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0230c9e34c63474799fa8d2c11fe3bbd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733465770333 2024-12-06T06:16:12,243 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:12,243 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#A#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:12,243 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/f24b74aaf642472193df0b1f8f3684bb is 50, key is test_row_0/B:col10/1733465769126/Put/seqid=0 2024-12-06T06:16:12,244 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/06c6109835194186b8c55307ce29fb2b is 50, key is test_row_0/A:col10/1733465769126/Put/seqid=0 2024-12-06T06:16:12,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742377_1553 (size=13017) 2024-12-06T06:16:12,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742378_1554 (size=13017) 2024-12-06T06:16:12,266 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/f24b74aaf642472193df0b1f8f3684bb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/f24b74aaf642472193df0b1f8f3684bb 2024-12-06T06:16:12,270 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/B of 6da2c4ed30e0a36b951d14c383b1d513 into f24b74aaf642472193df0b1f8f3684bb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:12,270 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:12,270 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/B, priority=12, startTime=1733465772234; duration=0sec 2024-12-06T06:16:12,270 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:12,270 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:B 2024-12-06T06:16:12,270 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:12,271 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:12,271 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): 6da2c4ed30e0a36b951d14c383b1d513/C is initiating minor compaction (all files) 2024-12-06T06:16:12,271 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6da2c4ed30e0a36b951d14c383b1d513/C in TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:12,271 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/177035299f3047f7822fe71231b94bd7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/015a0feb1348440a9b7a53ec7192daa0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9050045811f74ec1be5a574fdf44c98d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ab4f8f160a2e46ff812a5784eeecf766] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp, totalSize=46.2 K 2024-12-06T06:16:12,271 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 177035299f3047f7822fe71231b94bd7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733465767824 2024-12-06T06:16:12,271 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 015a0feb1348440a9b7a53ec7192daa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733465767867 2024-12-06T06:16:12,271 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9050045811f74ec1be5a574fdf44c98d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733465769108 2024-12-06T06:16:12,272 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting ab4f8f160a2e46ff812a5784eeecf766, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733465770333 2024-12-06T06:16:12,281 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6da2c4ed30e0a36b951d14c383b1d513#C#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:12,281 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/08a92bf3c5904782b1faf4bb45752dcd is 50, key is test_row_0/C:col10/1733465769126/Put/seqid=0 2024-12-06T06:16:12,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742379_1555 (size=13017) 2024-12-06T06:16:12,289 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/08a92bf3c5904782b1faf4bb45752dcd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/08a92bf3c5904782b1faf4bb45752dcd 2024-12-06T06:16:12,292 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/C of 6da2c4ed30e0a36b951d14c383b1d513 into 08a92bf3c5904782b1faf4bb45752dcd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:12,293 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:12,293 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/C, priority=12, startTime=1733465772234; duration=0sec 2024-12-06T06:16:12,293 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:12,293 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:C 2024-12-06T06:16:12,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-06T06:16:12,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:12,353 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-06T06:16:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:12,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/51da16bdcdf64cae87b5a33d65a2f0a8 is 50, key is test_row_0/A:col10/1733465771040/Put/seqid=0 2024-12-06T06:16:12,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742380_1556 (size=12301) 2024-12-06T06:16:12,368 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/51da16bdcdf64cae87b5a33d65a2f0a8 2024-12-06T06:16:12,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/98fad3251505494ea6f35daabfe1c8af is 50, key is test_row_0/B:col10/1733465771040/Put/seqid=0 2024-12-06T06:16:12,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742381_1557 (size=12301) 2024-12-06T06:16:12,658 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/06c6109835194186b8c55307ce29fb2b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/06c6109835194186b8c55307ce29fb2b 2024-12-06T06:16:12,662 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6da2c4ed30e0a36b951d14c383b1d513/A of 6da2c4ed30e0a36b951d14c383b1d513 into 06c6109835194186b8c55307ce29fb2b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:12,662 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:12,662 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., storeName=6da2c4ed30e0a36b951d14c383b1d513/A, priority=12, startTime=1733465772234; duration=0sec 2024-12-06T06:16:12,662 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:12,662 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6da2c4ed30e0a36b951d14c383b1d513:A 2024-12-06T06:16:12,780 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/98fad3251505494ea6f35daabfe1c8af 2024-12-06T06:16:12,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/9b3e0aea498e4e24aa4bbf721d0e0618 is 50, key is test_row_0/C:col10/1733465771040/Put/seqid=0 2024-12-06T06:16:12,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742382_1558 (size=12301) 2024-12-06T06:16:13,046 INFO [Thread-2197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2197. 2024-12-06T06:16:13,047 DEBUG [Thread-2197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:13,047 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,047 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:13,050 INFO [Thread-2193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2193. 2024-12-06T06:16:13,050 DEBUG [Thread-2193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:13,050 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,050 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:13,052 INFO [Thread-2189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2189. 2024-12-06T06:16:13,052 DEBUG [Thread-2189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:13,052 DEBUG [Thread-2189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,053 DEBUG [Thread-2189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:13,054 INFO [Thread-2195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2195. 2024-12-06T06:16:13,054 DEBUG [Thread-2195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:13,054 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,054 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:13,055 INFO [Thread-2191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2191. 2024-12-06T06:16:13,055 DEBUG [Thread-2191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:13,055 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,055 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:13,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-06T06:16:13,227 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/9b3e0aea498e4e24aa4bbf721d0e0618 2024-12-06T06:16:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/51da16bdcdf64cae87b5a33d65a2f0a8 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/51da16bdcdf64cae87b5a33d65a2f0a8 2024-12-06T06:16:13,233 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/51da16bdcdf64cae87b5a33d65a2f0a8, entries=150, sequenceid=512, filesize=12.0 K 2024-12-06T06:16:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/98fad3251505494ea6f35daabfe1c8af as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/98fad3251505494ea6f35daabfe1c8af 2024-12-06T06:16:13,236 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/98fad3251505494ea6f35daabfe1c8af, entries=150, sequenceid=512, filesize=12.0 K 2024-12-06T06:16:13,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/9b3e0aea498e4e24aa4bbf721d0e0618 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9b3e0aea498e4e24aa4bbf721d0e0618 2024-12-06T06:16:13,239 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9b3e0aea498e4e24aa4bbf721d0e0618, entries=150, sequenceid=512, filesize=12.0 K 2024-12-06T06:16:13,239 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 6da2c4ed30e0a36b951d14c383b1d513 in 887ms, sequenceid=512, compaction requested=false 2024-12-06T06:16:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 6da2c4ed30e0a36b951d14c383b1d513: 2024-12-06T06:16:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-06T06:16:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-06T06:16:13,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-06T06:16:13,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-12-06T06:16:13,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.2690 sec 2024-12-06T06:16:13,245 INFO [Thread-2186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2186. 2024-12-06T06:16:13,245 INFO [Thread-2184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2184. 2024-12-06T06:16:13,245 DEBUG [Thread-2186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:13,245 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,245 DEBUG [Thread-2184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:13,245 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,245 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,245 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:13,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:13,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-06T06:16:15,144 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:15,995 INFO [Thread-2182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2182. 2024-12-06T06:16:15,995 INFO [Thread-2180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2180. 2024-12-06T06:16:15,995 DEBUG [Thread-2182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:15,995 DEBUG [Thread-2180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:15,995 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:15,995 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:15,995 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:15,995 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:15,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:15,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:16,005 INFO [Thread-2178 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2178. 2024-12-06T06:16:16,005 DEBUG [Thread-2178 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:16,005 DEBUG [Thread-2178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:16,005 DEBUG [Thread-2178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:16,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 156 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 175 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1101 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3296 rows 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1103 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3306 rows 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1105 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3315 rows 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3308 rows 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1103 2024-12-06T06:16:16,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3306 rows 2024-12-06T06:16:16,005 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T06:16:16,005 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:16:16,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:16,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:16,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:16,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T06:16:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T06:16:16,008 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465776008"}]},"ts":"1733465776008"} 2024-12-06T06:16:16,009 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T06:16:16,010 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T06:16:16,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:16:16,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, UNASSIGN}] 2024-12-06T06:16:16,012 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, UNASSIGN 2024-12-06T06:16:16,012 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=6da2c4ed30e0a36b951d14c383b1d513, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:16,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, UNASSIGN because future has completed 2024-12-06T06:16:16,013 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:16:16,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:16:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T06:16:16,165 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:16,165 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 6da2c4ed30e0a36b951d14c383b1d513, disabling compactions & flushes 2024-12-06T06:16:16,166 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. after waiting 0 ms 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:16,166 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 6da2c4ed30e0a36b951d14c383b1d513 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=A 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=B 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6da2c4ed30e0a36b951d14c383b1d513, store=C 2024-12-06T06:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:16,169 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/3ca70f6bd128422e8146b56273506ca7 is 50, key is test_row_0/A:col10/1733465776004/Put/seqid=0 2024-12-06T06:16:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742383_1559 (size=12301) 2024-12-06T06:16:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T06:16:16,573 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/3ca70f6bd128422e8146b56273506ca7 2024-12-06T06:16:16,579 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/0286b65eb6d345e1bf12c07d8fa5f1d9 is 50, key is test_row_0/B:col10/1733465776004/Put/seqid=0 2024-12-06T06:16:16,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742384_1560 (size=12301) 2024-12-06T06:16:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T06:16:16,982 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/0286b65eb6d345e1bf12c07d8fa5f1d9 2024-12-06T06:16:16,987 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/84f81b1d20db4a13a1107b5702fc383b is 50, key is test_row_0/C:col10/1733465776004/Put/seqid=0 2024-12-06T06:16:16,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742385_1561 (size=12301) 2024-12-06T06:16:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T06:16:17,390 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/84f81b1d20db4a13a1107b5702fc383b 2024-12-06T06:16:17,394 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/A/3ca70f6bd128422e8146b56273506ca7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/3ca70f6bd128422e8146b56273506ca7 2024-12-06T06:16:17,396 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/3ca70f6bd128422e8146b56273506ca7, entries=150, sequenceid=521, filesize=12.0 K 2024-12-06T06:16:17,397 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/B/0286b65eb6d345e1bf12c07d8fa5f1d9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0286b65eb6d345e1bf12c07d8fa5f1d9 2024-12-06T06:16:17,399 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0286b65eb6d345e1bf12c07d8fa5f1d9, entries=150, sequenceid=521, filesize=12.0 K 2024-12-06T06:16:17,400 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/.tmp/C/84f81b1d20db4a13a1107b5702fc383b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/84f81b1d20db4a13a1107b5702fc383b 2024-12-06T06:16:17,402 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/84f81b1d20db4a13a1107b5702fc383b, entries=150, sequenceid=521, filesize=12.0 K 2024-12-06T06:16:17,403 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6da2c4ed30e0a36b951d14c383b1d513 in 1237ms, sequenceid=521, compaction requested=true 2024-12-06T06:16:17,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cfe0c9b1a0664ed39f78e8fa713b98c9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/84cb6038147840528dfc304b7a521ecf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f48261375cd243dca4fa231446fc98ff, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/89d63734460d45688aae6c6739d1dec0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c1b5669aa93f41c7b721db84da2e4805, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/e052fb7f9f8642fcaaa4dda0eda31ba1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/1b90b52052e74683b299d068fc1ad8af, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/8ad5d490a1524bed99d104d51cf8b758, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/5c1d5e900e17442aa1f8c7c8bd44f6b1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/763d6eb10af94de8b7f77c79aebd83b2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/da193b332d0a47d5b0bf0b0b28aa34eb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f680783d375e41cfb79c9f3b42752e0b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c4416e0805534dc2b48ac67920ad644a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/968bb57f274e4c978b17e02c6f376bd4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/d4aa9c3180514bcbb6910e8fe818d70c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/fca8f889bfda4025a3b8f1710594c3dc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/58ef2b84c98f4ec190f743197297370e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/737b41df4c9b495b8eaf26a218309be5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cf1c41fa2907491d86edf9bbc17a5ab9, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/ba55eb06105c42bf9638ac5d5c350c8f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/73c624c8386d4b28b0b9c82c417c784c] to archive 2024-12-06T06:16:17,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:16:17,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cfe0c9b1a0664ed39f78e8fa713b98c9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cfe0c9b1a0664ed39f78e8fa713b98c9 2024-12-06T06:16:17,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/84cb6038147840528dfc304b7a521ecf to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/84cb6038147840528dfc304b7a521ecf 2024-12-06T06:16:17,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f48261375cd243dca4fa231446fc98ff to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f48261375cd243dca4fa231446fc98ff 2024-12-06T06:16:17,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/89d63734460d45688aae6c6739d1dec0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/89d63734460d45688aae6c6739d1dec0 2024-12-06T06:16:17,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c1b5669aa93f41c7b721db84da2e4805 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c1b5669aa93f41c7b721db84da2e4805 2024-12-06T06:16:17,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/e052fb7f9f8642fcaaa4dda0eda31ba1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/e052fb7f9f8642fcaaa4dda0eda31ba1 2024-12-06T06:16:17,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/1b90b52052e74683b299d068fc1ad8af to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/1b90b52052e74683b299d068fc1ad8af 2024-12-06T06:16:17,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/8ad5d490a1524bed99d104d51cf8b758 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/8ad5d490a1524bed99d104d51cf8b758 2024-12-06T06:16:17,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/5c1d5e900e17442aa1f8c7c8bd44f6b1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/5c1d5e900e17442aa1f8c7c8bd44f6b1 2024-12-06T06:16:17,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/763d6eb10af94de8b7f77c79aebd83b2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/763d6eb10af94de8b7f77c79aebd83b2 2024-12-06T06:16:17,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/da193b332d0a47d5b0bf0b0b28aa34eb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/da193b332d0a47d5b0bf0b0b28aa34eb 2024-12-06T06:16:17,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f680783d375e41cfb79c9f3b42752e0b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/f680783d375e41cfb79c9f3b42752e0b 2024-12-06T06:16:17,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c4416e0805534dc2b48ac67920ad644a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/c4416e0805534dc2b48ac67920ad644a 2024-12-06T06:16:17,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/968bb57f274e4c978b17e02c6f376bd4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/968bb57f274e4c978b17e02c6f376bd4 2024-12-06T06:16:17,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/d4aa9c3180514bcbb6910e8fe818d70c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/d4aa9c3180514bcbb6910e8fe818d70c 2024-12-06T06:16:17,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/fca8f889bfda4025a3b8f1710594c3dc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/fca8f889bfda4025a3b8f1710594c3dc 2024-12-06T06:16:17,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/58ef2b84c98f4ec190f743197297370e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/58ef2b84c98f4ec190f743197297370e 2024-12-06T06:16:17,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/737b41df4c9b495b8eaf26a218309be5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/737b41df4c9b495b8eaf26a218309be5 2024-12-06T06:16:17,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cf1c41fa2907491d86edf9bbc17a5ab9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/cf1c41fa2907491d86edf9bbc17a5ab9 2024-12-06T06:16:17,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/ba55eb06105c42bf9638ac5d5c350c8f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/ba55eb06105c42bf9638ac5d5c350c8f 2024-12-06T06:16:17,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/73c624c8386d4b28b0b9c82c417c784c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/73c624c8386d4b28b0b9c82c417c784c 2024-12-06T06:16:17,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/fc6fdacac8ff44fdbaef1b003e8b41d2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/edf43a6e96cb410892c3a103dc69393b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/8f5fea87dac14a19ab72175b08ea0b08, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/24e247e3a11b41bebb916075857343cf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/ac3dccb8e1e142779f7b4822b844b2e7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/c97467280016466fa00c2be9b53e09b2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/d2eed472b9bc472298d6aa5949553b98, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e812353ecf5d40e2bce78f4ccb8693bd, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/277f79dea99d43ec8811d2288fb9716d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e6745b2e143e47fabb7b8a63a66589a4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/196243bcd2114aecb466fbdfd765e3e8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/4286a1e86d2d405c97116d133ebaf1d6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/cb3b1ca7423642efbad7fc13d53b1262, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/93e5feca64f64d088e494fb97949c095, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/72761a0ed92b4447adee5c3ac6ee68e5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/79c5aacec50a43efb3df454009513ee3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/462293e4442f4da39d3176707e607ddf, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/db327e81d3824696b54aa0b69d32ccdb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/88e0468e6bfc40fe9399b452563f0d1f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/2c61d846c296425594a854001866daa8, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0230c9e34c63474799fa8d2c11fe3bbd] to archive 2024-12-06T06:16:17,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:16:17,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/fc6fdacac8ff44fdbaef1b003e8b41d2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/fc6fdacac8ff44fdbaef1b003e8b41d2 2024-12-06T06:16:17,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/edf43a6e96cb410892c3a103dc69393b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/edf43a6e96cb410892c3a103dc69393b 2024-12-06T06:16:17,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/8f5fea87dac14a19ab72175b08ea0b08 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/8f5fea87dac14a19ab72175b08ea0b08 2024-12-06T06:16:17,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/24e247e3a11b41bebb916075857343cf to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/24e247e3a11b41bebb916075857343cf 2024-12-06T06:16:17,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/ac3dccb8e1e142779f7b4822b844b2e7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/ac3dccb8e1e142779f7b4822b844b2e7 2024-12-06T06:16:17,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/c97467280016466fa00c2be9b53e09b2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/c97467280016466fa00c2be9b53e09b2 2024-12-06T06:16:17,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/d2eed472b9bc472298d6aa5949553b98 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/d2eed472b9bc472298d6aa5949553b98 2024-12-06T06:16:17,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e812353ecf5d40e2bce78f4ccb8693bd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e812353ecf5d40e2bce78f4ccb8693bd 2024-12-06T06:16:17,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/277f79dea99d43ec8811d2288fb9716d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/277f79dea99d43ec8811d2288fb9716d 2024-12-06T06:16:17,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e6745b2e143e47fabb7b8a63a66589a4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/e6745b2e143e47fabb7b8a63a66589a4 2024-12-06T06:16:17,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/196243bcd2114aecb466fbdfd765e3e8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/196243bcd2114aecb466fbdfd765e3e8 2024-12-06T06:16:17,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/4286a1e86d2d405c97116d133ebaf1d6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/4286a1e86d2d405c97116d133ebaf1d6 2024-12-06T06:16:17,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/cb3b1ca7423642efbad7fc13d53b1262 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/cb3b1ca7423642efbad7fc13d53b1262 2024-12-06T06:16:17,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/93e5feca64f64d088e494fb97949c095 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/93e5feca64f64d088e494fb97949c095 2024-12-06T06:16:17,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/72761a0ed92b4447adee5c3ac6ee68e5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/72761a0ed92b4447adee5c3ac6ee68e5 2024-12-06T06:16:17,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/79c5aacec50a43efb3df454009513ee3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/79c5aacec50a43efb3df454009513ee3 2024-12-06T06:16:17,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/462293e4442f4da39d3176707e607ddf to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/462293e4442f4da39d3176707e607ddf 2024-12-06T06:16:17,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/db327e81d3824696b54aa0b69d32ccdb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/db327e81d3824696b54aa0b69d32ccdb 2024-12-06T06:16:17,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/88e0468e6bfc40fe9399b452563f0d1f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/88e0468e6bfc40fe9399b452563f0d1f 2024-12-06T06:16:17,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/2c61d846c296425594a854001866daa8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/2c61d846c296425594a854001866daa8 2024-12-06T06:16:17,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0230c9e34c63474799fa8d2c11fe3bbd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0230c9e34c63474799fa8d2c11fe3bbd 2024-12-06T06:16:17,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/6519c805741842c9b92b9f67be1e7040, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/4771616e65ef433198d627464db2055c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7af225dad3f043b3a4a6c8a4a7ac2924, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/c62f735c2fc840d4951d8fbe5eae3737, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/0843b41ed259461db1ba7d964e5b1389, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7e79f7fbc7a64c91864a3d73b00db2e6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/215ee22d87fe4945b27a0e5650bcd095, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/5e625ad0fbfa4943898d021acc67ab1d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b914d7ae8197447889e2cb951baac0f4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f71e70b293804899b36724b9679da993, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b4d192df39bf4020ba6db9fbc6670a96, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/137e90200a634c4fb581fc17434451ad, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ea3816c887a043d7abb32ee1937af5e0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/3513fba61ca44f22823e8d9b5c8638b5, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/27da05b979d34a17b52ecadc707bbe2d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/21a01204d93b4e50b00095a0e5af4562, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/177035299f3047f7822fe71231b94bd7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f6b506ad080e4046b6a35550e98eb135, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/015a0feb1348440a9b7a53ec7192daa0, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9050045811f74ec1be5a574fdf44c98d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ab4f8f160a2e46ff812a5784eeecf766] to archive 2024-12-06T06:16:17,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:16:17,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/6519c805741842c9b92b9f67be1e7040 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/6519c805741842c9b92b9f67be1e7040 2024-12-06T06:16:17,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/4771616e65ef433198d627464db2055c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/4771616e65ef433198d627464db2055c 2024-12-06T06:16:17,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7af225dad3f043b3a4a6c8a4a7ac2924 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7af225dad3f043b3a4a6c8a4a7ac2924 2024-12-06T06:16:17,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/c62f735c2fc840d4951d8fbe5eae3737 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/c62f735c2fc840d4951d8fbe5eae3737 2024-12-06T06:16:17,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/0843b41ed259461db1ba7d964e5b1389 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/0843b41ed259461db1ba7d964e5b1389 2024-12-06T06:16:17,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7e79f7fbc7a64c91864a3d73b00db2e6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/7e79f7fbc7a64c91864a3d73b00db2e6 2024-12-06T06:16:17,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/215ee22d87fe4945b27a0e5650bcd095 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/215ee22d87fe4945b27a0e5650bcd095 2024-12-06T06:16:17,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/5e625ad0fbfa4943898d021acc67ab1d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/5e625ad0fbfa4943898d021acc67ab1d 2024-12-06T06:16:17,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b914d7ae8197447889e2cb951baac0f4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b914d7ae8197447889e2cb951baac0f4 2024-12-06T06:16:17,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f71e70b293804899b36724b9679da993 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f71e70b293804899b36724b9679da993 2024-12-06T06:16:17,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b4d192df39bf4020ba6db9fbc6670a96 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/b4d192df39bf4020ba6db9fbc6670a96 2024-12-06T06:16:17,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/137e90200a634c4fb581fc17434451ad to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/137e90200a634c4fb581fc17434451ad 2024-12-06T06:16:17,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ea3816c887a043d7abb32ee1937af5e0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ea3816c887a043d7abb32ee1937af5e0 2024-12-06T06:16:17,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/3513fba61ca44f22823e8d9b5c8638b5 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/3513fba61ca44f22823e8d9b5c8638b5 2024-12-06T06:16:17,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/27da05b979d34a17b52ecadc707bbe2d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/27da05b979d34a17b52ecadc707bbe2d 2024-12-06T06:16:17,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/21a01204d93b4e50b00095a0e5af4562 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/21a01204d93b4e50b00095a0e5af4562 2024-12-06T06:16:17,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/177035299f3047f7822fe71231b94bd7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/177035299f3047f7822fe71231b94bd7 2024-12-06T06:16:17,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f6b506ad080e4046b6a35550e98eb135 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/f6b506ad080e4046b6a35550e98eb135 2024-12-06T06:16:17,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/015a0feb1348440a9b7a53ec7192daa0 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/015a0feb1348440a9b7a53ec7192daa0 2024-12-06T06:16:17,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9050045811f74ec1be5a574fdf44c98d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9050045811f74ec1be5a574fdf44c98d 2024-12-06T06:16:17,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ab4f8f160a2e46ff812a5784eeecf766 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/ab4f8f160a2e46ff812a5784eeecf766 2024-12-06T06:16:17,461 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/recovered.edits/524.seqid, newMaxSeqId=524, maxSeqId=1 2024-12-06T06:16:17,461 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. 2024-12-06T06:16:17,461 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 6da2c4ed30e0a36b951d14c383b1d513: Waiting for close lock at 1733465776166Running coprocessor pre-close hooks at 1733465776166Disabling compacts and flushes for region at 1733465776166Disabling writes for close at 1733465776166Obtaining lock to block concurrent updates at 1733465776166Preparing flush snapshotting stores in 6da2c4ed30e0a36b951d14c383b1d513 at 1733465776166Finished memstore snapshotting TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733465776166Flushing stores of TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513. at 1733465776167 (+1 ms)Flushing 6da2c4ed30e0a36b951d14c383b1d513/A: creating writer at 1733465776167Flushing 6da2c4ed30e0a36b951d14c383b1d513/A: appending metadata at 1733465776169 (+2 ms)Flushing 6da2c4ed30e0a36b951d14c383b1d513/A: closing flushed file at 1733465776169Flushing 6da2c4ed30e0a36b951d14c383b1d513/B: creating writer at 1733465776576 (+407 ms)Flushing 6da2c4ed30e0a36b951d14c383b1d513/B: appending metadata at 1733465776578 (+2 ms)Flushing 6da2c4ed30e0a36b951d14c383b1d513/B: closing flushed file at 1733465776578Flushing 6da2c4ed30e0a36b951d14c383b1d513/C: creating writer at 1733465776985 (+407 ms)Flushing 6da2c4ed30e0a36b951d14c383b1d513/C: appending metadata at 1733465776987 (+2 ms)Flushing 6da2c4ed30e0a36b951d14c383b1d513/C: closing flushed file at 1733465776987Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d8a6ef1: reopening flushed file at 1733465777393 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31625946: reopening flushed file at 1733465777397 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16f238a0: reopening flushed file at 1733465777399 (+2 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6da2c4ed30e0a36b951d14c383b1d513 in 1237ms, sequenceid=521, compaction requested=true at 1733465777403 (+4 ms)Writing region close event to WAL at 1733465777458 (+55 ms)Running coprocessor post-close hooks at 1733465777461 (+3 ms)Closed at 1733465777461 2024-12-06T06:16:17,463 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:17,463 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=6da2c4ed30e0a36b951d14c383b1d513, regionState=CLOSED 2024-12-06T06:16:17,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:16:17,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-06T06:16:17,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 6da2c4ed30e0a36b951d14c383b1d513, server=6336e8474e1c,39987,1733465620995 in 1.4520 sec 2024-12-06T06:16:17,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-06T06:16:17,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6da2c4ed30e0a36b951d14c383b1d513, UNASSIGN in 1.4550 sec 2024-12-06T06:16:17,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-06T06:16:17,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4580 sec 2024-12-06T06:16:17,470 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465777470"}]},"ts":"1733465777470"} 2024-12-06T06:16:17,471 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T06:16:17,471 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T06:16:17,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4660 sec 2024-12-06T06:16:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T06:16:18,154 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:18,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T06:16:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:18,156 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-06T06:16:18,157 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:18,158 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:18,160 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/recovered.edits] 2024-12-06T06:16:18,162 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/06c6109835194186b8c55307ce29fb2b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/06c6109835194186b8c55307ce29fb2b 2024-12-06T06:16:18,162 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/3ca70f6bd128422e8146b56273506ca7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/3ca70f6bd128422e8146b56273506ca7 2024-12-06T06:16:18,163 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/51da16bdcdf64cae87b5a33d65a2f0a8 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/A/51da16bdcdf64cae87b5a33d65a2f0a8 2024-12-06T06:16:18,165 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0286b65eb6d345e1bf12c07d8fa5f1d9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/0286b65eb6d345e1bf12c07d8fa5f1d9 2024-12-06T06:16:18,165 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/98fad3251505494ea6f35daabfe1c8af to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/98fad3251505494ea6f35daabfe1c8af 2024-12-06T06:16:18,166 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/f24b74aaf642472193df0b1f8f3684bb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/B/f24b74aaf642472193df0b1f8f3684bb 2024-12-06T06:16:18,167 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/08a92bf3c5904782b1faf4bb45752dcd to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/08a92bf3c5904782b1faf4bb45752dcd 2024-12-06T06:16:18,168 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/84f81b1d20db4a13a1107b5702fc383b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/84f81b1d20db4a13a1107b5702fc383b 2024-12-06T06:16:18,169 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9b3e0aea498e4e24aa4bbf721d0e0618 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/C/9b3e0aea498e4e24aa4bbf721d0e0618 2024-12-06T06:16:18,170 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/recovered.edits/524.seqid to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513/recovered.edits/524.seqid 2024-12-06T06:16:18,171 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/6da2c4ed30e0a36b951d14c383b1d513 2024-12-06T06:16:18,171 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T06:16:18,172 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:18,174 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T06:16:18,175 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T06:16:18,176 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:18,176 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T06:16:18,177 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733465778176"}]},"ts":"9223372036854775807"} 2024-12-06T06:16:18,178 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T06:16:18,178 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6da2c4ed30e0a36b951d14c383b1d513, NAME => 'TestAcidGuarantees,,1733465750844.6da2c4ed30e0a36b951d14c383b1d513.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T06:16:18,178 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T06:16:18,178 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733465778178"}]},"ts":"9223372036854775807"} 2024-12-06T06:16:18,179 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T06:16:18,180 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:18,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 25 msec 2024-12-06T06:16:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-06T06:16:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T06:16:18,266 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:18,273 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=443 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=347 (was 342) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4235 (was 4321) 2024-12-06T06:16:18,281 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=347, ProcessCount=11, AvailableMemoryMB=4235 2024-12-06T06:16:18,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:16:18,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:16:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:18,284 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T06:16:18,284 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:18,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-06T06:16:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T06:16:18,285 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T06:16:18,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742386_1562 (size=1061) 2024-12-06T06:16:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T06:16:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T06:16:18,691 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea 2024-12-06T06:16:18,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742387_1563 (size=53) 2024-12-06T06:16:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T06:16:19,095 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:16:19,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b41c24c7a57a1a19a0dc6ea2edf56813, disabling compactions & flushes 2024-12-06T06:16:19,096 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:19,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:19,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. after waiting 0 ms 2024-12-06T06:16:19,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:19,096 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:19,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b41c24c7a57a1a19a0dc6ea2edf56813: Waiting for close lock at 1733465779096Disabling compacts and flushes for region at 1733465779096Disabling writes for close at 1733465779096Writing region close event to WAL at 1733465779096Closed at 1733465779096 2024-12-06T06:16:19,097 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T06:16:19,097 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733465779097"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733465779097"}]},"ts":"1733465779097"} 2024-12-06T06:16:19,098 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T06:16:19,099 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T06:16:19,099 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465779099"}]},"ts":"1733465779099"} 2024-12-06T06:16:19,100 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T06:16:19,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, ASSIGN}] 2024-12-06T06:16:19,101 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, ASSIGN 2024-12-06T06:16:19,102 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, ASSIGN; state=OFFLINE, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=false 2024-12-06T06:16:19,252 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=b41c24c7a57a1a19a0dc6ea2edf56813, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:19,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, ASSIGN because future has completed 2024-12-06T06:16:19,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:16:19,407 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:19,408 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:16:19,408 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,408 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:16:19,408 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,408 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,409 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,410 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:16:19,410 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b41c24c7a57a1a19a0dc6ea2edf56813 columnFamilyName A 2024-12-06T06:16:19,410 DEBUG [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:19,411 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(327): Store=b41c24c7a57a1a19a0dc6ea2edf56813/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:16:19,411 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,412 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:16:19,412 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b41c24c7a57a1a19a0dc6ea2edf56813 columnFamilyName B 2024-12-06T06:16:19,412 DEBUG [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:19,412 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(327): Store=b41c24c7a57a1a19a0dc6ea2edf56813/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:16:19,412 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,413 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:16:19,413 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b41c24c7a57a1a19a0dc6ea2edf56813 columnFamilyName C 2024-12-06T06:16:19,413 DEBUG [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:19,413 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(327): Store=b41c24c7a57a1a19a0dc6ea2edf56813/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:16:19,413 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:19,413 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,414 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,414 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,415 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,415 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,415 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:16:19,416 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,418 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T06:16:19,418 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened b41c24c7a57a1a19a0dc6ea2edf56813; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63770495, jitterRate=-0.049745574593544006}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:16:19,418 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:19,418 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for b41c24c7a57a1a19a0dc6ea2edf56813: Running coprocessor pre-open hook at 1733465779408Writing region info on filesystem at 1733465779408Initializing all the Stores at 1733465779409 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465779409Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465779409Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465779409Cleaning up temporary data from old regions at 1733465779415 (+6 ms)Running coprocessor post-open hooks at 1733465779418 (+3 ms)Region opened successfully at 1733465779418 2024-12-06T06:16:19,419 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., pid=144, masterSystemTime=1733465779405 2024-12-06T06:16:19,421 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:19,421 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:19,421 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=b41c24c7a57a1a19a0dc6ea2edf56813, regionState=OPEN, openSeqNum=2, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:19,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:16:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T06:16:19,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-06T06:16:19,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 in 169 msec 2024-12-06T06:16:19,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-06T06:16:19,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, ASSIGN in 325 msec 2024-12-06T06:16:19,427 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T06:16:19,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465779427"}]},"ts":"1733465779427"} 2024-12-06T06:16:19,428 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T06:16:19,428 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T06:16:19,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-12-06T06:16:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T06:16:20,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:20,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:20,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:20,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:20,436 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:20,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:20,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fac2c87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:20,438 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ada9d85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:20,438 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:20,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:20,440 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T06:16:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 6336e8474e1c,40959,1733465620259 2024-12-06T06:16:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c0978e6 2024-12-06T06:16:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T06:16:20,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T06:16:20,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T06:16:20,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T06:16:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:20,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742388_1564 (size=1097) 2024-12-06T06:16:20,852 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-06T06:16:20,853 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-06T06:16:20,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:16:20,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, REOPEN/MOVE}] 2024-12-06T06:16:20,857 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, REOPEN/MOVE 2024-12-06T06:16:20,858 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=b41c24c7a57a1a19a0dc6ea2edf56813, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:20,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, REOPEN/MOVE because future has completed 2024-12-06T06:16:20,859 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:16:20,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:16:21,011 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,011 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:16:21,011 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing b41c24c7a57a1a19a0dc6ea2edf56813, disabling compactions & flushes 2024-12-06T06:16:21,011 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,011 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,011 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. after waiting 0 ms 2024-12-06T06:16:21,011 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,014 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T06:16:21,015 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,015 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for b41c24c7a57a1a19a0dc6ea2edf56813: Waiting for close lock at 1733465781011Running coprocessor pre-close hooks at 1733465781011Disabling compacts and flushes for region at 1733465781011Disabling writes for close at 1733465781011Writing region close event to WAL at 1733465781012 (+1 ms)Running coprocessor post-close hooks at 1733465781015 (+3 ms)Closed at 1733465781015 2024-12-06T06:16:21,015 WARN [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: b41c24c7a57a1a19a0dc6ea2edf56813 to self. 2024-12-06T06:16:21,016 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,017 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=b41c24c7a57a1a19a0dc6ea2edf56813, regionState=CLOSED 2024-12-06T06:16:21,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:16:21,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-06T06:16:21,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 in 159 msec 2024-12-06T06:16:21,020 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, REOPEN/MOVE; state=CLOSED, location=6336e8474e1c,39987,1733465620995; forceNewPlan=false, retain=true 2024-12-06T06:16:21,171 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=b41c24c7a57a1a19a0dc6ea2edf56813, regionState=OPENING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, REOPEN/MOVE because future has completed 2024-12-06T06:16:21,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:16:21,326 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,327 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} 2024-12-06T06:16:21,327 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,327 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T06:16:21,327 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,327 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,328 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,328 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:16:21,329 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b41c24c7a57a1a19a0dc6ea2edf56813 columnFamilyName A 2024-12-06T06:16:21,330 DEBUG [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:21,330 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(327): Store=b41c24c7a57a1a19a0dc6ea2edf56813/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:16:21,330 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,330 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:16:21,331 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b41c24c7a57a1a19a0dc6ea2edf56813 columnFamilyName B 2024-12-06T06:16:21,331 DEBUG [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:21,331 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(327): Store=b41c24c7a57a1a19a0dc6ea2edf56813/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:16:21,331 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,331 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-06T06:16:21,331 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b41c24c7a57a1a19a0dc6ea2edf56813 columnFamilyName C 2024-12-06T06:16:21,331 DEBUG [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:21,332 INFO [StoreOpener-b41c24c7a57a1a19a0dc6ea2edf56813-1 {}] regionserver.HStore(327): Store=b41c24c7a57a1a19a0dc6ea2edf56813/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T06:16:21,332 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,332 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,332 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,333 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,334 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,334 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,334 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T06:16:21,335 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,335 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened b41c24c7a57a1a19a0dc6ea2edf56813; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70107187, jitterRate=0.044678494334220886}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T06:16:21,335 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,336 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for b41c24c7a57a1a19a0dc6ea2edf56813: Running coprocessor pre-open hook at 1733465781327Writing region info on filesystem at 1733465781327Initializing all the Stores at 1733465781328 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465781328Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465781328Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733465781328Cleaning up temporary data from old regions at 1733465781334 (+6 ms)Running coprocessor post-open hooks at 1733465781335 (+1 ms)Region opened successfully at 1733465781336 (+1 ms) 2024-12-06T06:16:21,336 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., pid=149, masterSystemTime=1733465781324 2024-12-06T06:16:21,338 DEBUG [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,338 INFO [RS_OPEN_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,338 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=b41c24c7a57a1a19a0dc6ea2edf56813, regionState=OPEN, openSeqNum=5, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:16:21,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-06T06:16:21,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 in 168 msec 2024-12-06T06:16:21,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-06T06:16:21,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, REOPEN/MOVE in 485 msec 2024-12-06T06:16:21,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-06T06:16:21,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-12-06T06:16:21,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-12-06T06:16:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-06T06:16:21,347 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:21,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,348 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c298bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,349 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f6c243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,351 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc87e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,352 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343f8131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,353 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5341988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,355 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@611b7e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,356 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dedcb02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,357 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f7bf3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,359 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fcf61c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,360 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e6b210e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,361 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58577040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,362 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37d2335e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,364 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f77507f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,365 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b8f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,367 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a298783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,368 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6e59d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd1a81c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,370 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295e49f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,371 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fce999d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46005d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6336e8474e1c,40959,-1 for getting cluster id 2024-12-06T06:16:21,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T06:16:21,373 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8824d7b1-a9b8-45af-a59f-d90ecce0f1c1' 2024-12-06T06:16:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T06:16:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8824d7b1-a9b8-45af-a59f-d90ecce0f1c1" 2024-12-06T06:16:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5639c6f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6336e8474e1c,40959,-1] 2024-12-06T06:16:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T06:16:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:21,374 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T06:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7376d787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T06:16:21,375 DEBUG [Thread-2515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,375 DEBUG [Thread-2524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,375 DEBUG [Thread-2520 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,375 DEBUG [Thread-2526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,375 DEBUG [Thread-2518 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,376 DEBUG [Thread-2522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,376 DEBUG [Thread-2511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,377 DEBUG [Thread-2513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-06T06:16:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:21,393 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,400 DEBUG [Thread-2509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T06:16:21,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6336e8474e1c,39987,1733465620995, seqNum=-1] 2024-12-06T06:16:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T06:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-06T06:16:21,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T06:16:21,403 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T06:16:21,403 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:21,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5] 2024-12-06T06:16:21,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465781411, max timestamp=1733465781425Num uniques 859; 2024-12-06T06:16:21,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:21,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465781411, max timestamp=1733465781425Num uniques 859; 2024-12-06T06:16:21,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:21,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733465781411, max timestamp=1733465781425Num uniques 859; 2024-12-06T06:16:21,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6336e8474e1c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-06T06:16:21,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062b1e0beb15d24fc28541e9486fd8158b_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465781388/Put/seqid=0 2024-12-06T06:16:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465841452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465841454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465841456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465841457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465841458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742389_1565 (size=12154) 2024-12-06T06:16:21,471 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:21,475 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062b1e0beb15d24fc28541e9486fd8158b_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b1e0beb15d24fc28541e9486fd8158b_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:21,476 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/16f0bf2143744213ba3e22f535fc01d1, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:21,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/16f0bf2143744213ba3e22f535fc01d1 is 175, key is test_row_0/A:col10/1733465781388/Put/seqid=0 2024-12-06T06:16:21,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742390_1566 (size=30955) 2024-12-06T06:16:21,485 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/16f0bf2143744213ba3e22f535fc01d1 2024-12-06T06:16:21,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/96216d83ebc24ed6bee431d20b0a2d43 is 50, key is test_row_0/B:col10/1733465781388/Put/seqid=0 2024-12-06T06:16:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T06:16:21,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742391_1567 (size=12001) 2024-12-06T06:16:21,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T06:16:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465841564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465841564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465841573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465841573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465841573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T06:16:21,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:21,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T06:16:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465841774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465841774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465841783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465841783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465841784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:21,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T06:16:21,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:21,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:21,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:21,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/96216d83ebc24ed6bee431d20b0a2d43 2024-12-06T06:16:21,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/68ac8acce4024e2eb6df2aeb3745a503 is 50, key is test_row_0/C:col10/1733465781388/Put/seqid=0 2024-12-06T06:16:21,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742392_1568 (size=12001) 2024-12-06T06:16:22,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T06:16:22,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:22,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:22,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:22,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T06:16:22,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465842084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465842084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465842093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465842093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465842093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T06:16:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:22,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T06:16:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:22,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:22,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/68ac8acce4024e2eb6df2aeb3745a503 2024-12-06T06:16:22,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/16f0bf2143744213ba3e22f535fc01d1 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/16f0bf2143744213ba3e22f535fc01d1 2024-12-06T06:16:22,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/16f0bf2143744213ba3e22f535fc01d1, entries=150, sequenceid=14, filesize=30.2 K 2024-12-06T06:16:22,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/96216d83ebc24ed6bee431d20b0a2d43 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/96216d83ebc24ed6bee431d20b0a2d43 2024-12-06T06:16:22,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/96216d83ebc24ed6bee431d20b0a2d43, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T06:16:22,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/68ac8acce4024e2eb6df2aeb3745a503 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/68ac8acce4024e2eb6df2aeb3745a503 2024-12-06T06:16:22,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/68ac8acce4024e2eb6df2aeb3745a503, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T06:16:22,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b41c24c7a57a1a19a0dc6ea2edf56813 in 972ms, sequenceid=14, compaction requested=false 2024-12-06T06:16:22,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:22,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T06:16:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:22,472 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-06T06:16:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-06T06:16:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120626c66874fdc64c2db8f7afa4b6988951_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465781456/Put/seqid=0 2024-12-06T06:16:22,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742393_1569 (size=12154) 2024-12-06T06:16:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T06:16:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:22,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465842604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465842606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465842609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465842610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465842610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465842714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465842724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465842724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465842724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:22,888 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120626c66874fdc64c2db8f7afa4b6988951_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120626c66874fdc64c2db8f7afa4b6988951_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/11c5fa5f9d3246e9968ca51155a0c382, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:22,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/11c5fa5f9d3246e9968ca51155a0c382 is 175, key is test_row_0/A:col10/1733465781456/Put/seqid=0 2024-12-06T06:16:22,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742394_1570 (size=30955) 2024-12-06T06:16:22,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465842924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465842934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465842934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465842934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:22,977 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T06:16:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465843233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:23,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465843243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465843243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:23,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465843243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:23,293 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/11c5fa5f9d3246e9968ca51155a0c382 2024-12-06T06:16:23,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/f4716d3dcdc8410bba3776d022b00e3a is 50, key is test_row_0/B:col10/1733465781456/Put/seqid=0 2024-12-06T06:16:23,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742395_1571 (size=12001) 2024-12-06T06:16:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T06:16:23,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465843623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,626 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:23,703 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/f4716d3dcdc8410bba3776d022b00e3a 2024-12-06T06:16:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/0d51a5c859b149a093bb9ecdaedcc6ee is 50, key is test_row_0/C:col10/1733465781456/Put/seqid=0 2024-12-06T06:16:23,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742396_1572 (size=12001) 2024-12-06T06:16:23,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465843743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:23,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465843754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465843754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465843764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:24,114 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/0d51a5c859b149a093bb9ecdaedcc6ee 2024-12-06T06:16:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/11c5fa5f9d3246e9968ca51155a0c382 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/11c5fa5f9d3246e9968ca51155a0c382 2024-12-06T06:16:24,121 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/11c5fa5f9d3246e9968ca51155a0c382, entries=150, sequenceid=50, filesize=30.2 K 2024-12-06T06:16:24,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/f4716d3dcdc8410bba3776d022b00e3a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/f4716d3dcdc8410bba3776d022b00e3a 2024-12-06T06:16:24,125 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/f4716d3dcdc8410bba3776d022b00e3a, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T06:16:24,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/0d51a5c859b149a093bb9ecdaedcc6ee as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0d51a5c859b149a093bb9ecdaedcc6ee 2024-12-06T06:16:24,128 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0d51a5c859b149a093bb9ecdaedcc6ee, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T06:16:24,129 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1657ms, sequenceid=50, compaction requested=false 2024-12-06T06:16:24,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:24,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:24,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-06T06:16:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-06T06:16:24,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-06T06:16:24,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7280 sec 2024-12-06T06:16:24,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.7330 sec 2024-12-06T06:16:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:24,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-06T06:16:24,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:24,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:24,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:24,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:24,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:24,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:24,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062b3325ac8a4248aa80130098af384b67_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465782608/Put/seqid=0 2024-12-06T06:16:24,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742397_1573 (size=12154) 2024-12-06T06:16:24,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465844784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:24,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465844785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465844790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465844791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:24,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465844894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:24,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465844894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465844904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465844904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465845103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465845104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465845114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465845114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,169 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:25,172 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062b3325ac8a4248aa80130098af384b67_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b3325ac8a4248aa80130098af384b67_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:25,173 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/e1455a9e11fc42e9808c4b1fa7fc2028, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:25,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/e1455a9e11fc42e9808c4b1fa7fc2028 is 175, key is test_row_0/A:col10/1733465782608/Put/seqid=0 2024-12-06T06:16:25,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742398_1574 (size=30955) 2024-12-06T06:16:25,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/e1455a9e11fc42e9808c4b1fa7fc2028 2024-12-06T06:16:25,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/89c9b430e75a4aed84ba3d3c87c97154 is 50, key is test_row_0/B:col10/1733465782608/Put/seqid=0 2024-12-06T06:16:25,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742399_1575 (size=12001) 2024-12-06T06:16:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465845413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465845413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465845424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465845424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T06:16:25,564 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:25,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-06T06:16:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T06:16:25,566 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:25,567 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:25,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:25,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/89c9b430e75a4aed84ba3d3c87c97154 2024-12-06T06:16:25,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/6a49b06ca95c422994c0835603b44250 is 50, key is test_row_0/C:col10/1733465782608/Put/seqid=0 2024-12-06T06:16:25,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742400_1576 (size=12001) 2024-12-06T06:16:25,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465845643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,648 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T06:16:25,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T06:16:25,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:25,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:25,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:25,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:25,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:25,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T06:16:25,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:25,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:25,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:25,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:25,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T06:16:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465845924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465845924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465845933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465845933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:25,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/6a49b06ca95c422994c0835603b44250 2024-12-06T06:16:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/e1455a9e11fc42e9808c4b1fa7fc2028 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1455a9e11fc42e9808c4b1fa7fc2028 2024-12-06T06:16:26,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1455a9e11fc42e9808c4b1fa7fc2028, entries=150, sequenceid=62, filesize=30.2 K 2024-12-06T06:16:26,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/89c9b430e75a4aed84ba3d3c87c97154 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/89c9b430e75a4aed84ba3d3c87c97154 2024-12-06T06:16:26,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/89c9b430e75a4aed84ba3d3c87c97154, entries=150, sequenceid=62, filesize=11.7 K 2024-12-06T06:16:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/6a49b06ca95c422994c0835603b44250 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6a49b06ca95c422994c0835603b44250 2024-12-06T06:16:26,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6a49b06ca95c422994c0835603b44250, entries=150, sequenceid=62, filesize=11.7 K 2024-12-06T06:16:26,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1255ms, sequenceid=62, compaction requested=true 2024-12-06T06:16:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:26,012 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:26,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:26,012 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:26,013 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:26,013 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:26,013 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/A is initiating minor compaction (all files) 2024-12-06T06:16:26,013 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/B is initiating minor compaction (all files) 2024-12-06T06:16:26,013 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/A in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:26,013 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/B in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:26,013 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/16f0bf2143744213ba3e22f535fc01d1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/11c5fa5f9d3246e9968ca51155a0c382, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1455a9e11fc42e9808c4b1fa7fc2028] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=90.7 K 2024-12-06T06:16:26,013 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/96216d83ebc24ed6bee431d20b0a2d43, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/f4716d3dcdc8410bba3776d022b00e3a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/89c9b430e75a4aed84ba3d3c87c97154] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=35.2 K 2024-12-06T06:16:26,013 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:26,013 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/16f0bf2143744213ba3e22f535fc01d1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/11c5fa5f9d3246e9968ca51155a0c382, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1455a9e11fc42e9808c4b1fa7fc2028] 2024-12-06T06:16:26,014 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f0bf2143744213ba3e22f535fc01d1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465781380 2024-12-06T06:16:26,014 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 96216d83ebc24ed6bee431d20b0a2d43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465781380 2024-12-06T06:16:26,014 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting f4716d3dcdc8410bba3776d022b00e3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465781452 2024-12-06T06:16:26,014 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11c5fa5f9d3246e9968ca51155a0c382, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465781452 2024-12-06T06:16:26,014 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 89c9b430e75a4aed84ba3d3c87c97154, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733465782607 2024-12-06T06:16:26,015 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1455a9e11fc42e9808c4b1fa7fc2028, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733465782607 2024-12-06T06:16:26,020 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:26,022 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120624b8542e76984d8b81e84b6a7cbf6721_b41c24c7a57a1a19a0dc6ea2edf56813 store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:26,022 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:26,022 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/5ab66167b95e4ef39968598bfa6e5db6 is 50, key is test_row_0/B:col10/1733465782608/Put/seqid=0 2024-12-06T06:16:26,023 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120624b8542e76984d8b81e84b6a7cbf6721_b41c24c7a57a1a19a0dc6ea2edf56813, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:26,023 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120624b8542e76984d8b81e84b6a7cbf6721_b41c24c7a57a1a19a0dc6ea2edf56813 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:26,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T06:16:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:26,025 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-06T06:16:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:26,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742401_1577 (size=12104) 2024-12-06T06:16:26,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742402_1578 (size=4469) 2024-12-06T06:16:26,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f6e816a5c5014fb98faf260cfdc4e5af_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465784783/Put/seqid=0 2024-12-06T06:16:26,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742403_1579 (size=12154) 2024-12-06T06:16:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T06:16:26,436 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/5ab66167b95e4ef39968598bfa6e5db6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5ab66167b95e4ef39968598bfa6e5db6 2024-12-06T06:16:26,440 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:26,441 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/729cfb3dbfb8411ebd9120371ff3ccb6 is 175, key is test_row_0/A:col10/1733465782608/Put/seqid=0 2024-12-06T06:16:26,441 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/B of b41c24c7a57a1a19a0dc6ea2edf56813 into 5ab66167b95e4ef39968598bfa6e5db6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:26,441 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:26,442 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/B, priority=13, startTime=1733465786012; duration=0sec 2024-12-06T06:16:26,442 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:26,442 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:B 2024-12-06T06:16:26,442 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:26,443 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:26,443 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/C is initiating minor compaction (all files) 2024-12-06T06:16:26,443 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/C in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:26,443 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/68ac8acce4024e2eb6df2aeb3745a503, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0d51a5c859b149a093bb9ecdaedcc6ee, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6a49b06ca95c422994c0835603b44250] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=35.2 K 2024-12-06T06:16:26,443 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 68ac8acce4024e2eb6df2aeb3745a503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733465781380 2024-12-06T06:16:26,444 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d51a5c859b149a093bb9ecdaedcc6ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733465781452 2024-12-06T06:16:26,444 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a49b06ca95c422994c0835603b44250, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733465782607 2024-12-06T06:16:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742404_1580 (size=31058) 2024-12-06T06:16:26,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:26,451 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/729cfb3dbfb8411ebd9120371ff3ccb6 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/729cfb3dbfb8411ebd9120371ff3ccb6 2024-12-06T06:16:26,453 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:26,454 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/0f511a4927954126840e810acc72b64d is 50, key is test_row_0/C:col10/1733465782608/Put/seqid=0 2024-12-06T06:16:26,455 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f6e816a5c5014fb98faf260cfdc4e5af_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f6e816a5c5014fb98faf260cfdc4e5af_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:26,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/d5ac6ae428e84941b28767c3a34f8374, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/d5ac6ae428e84941b28767c3a34f8374 is 175, key is test_row_0/A:col10/1733465784783/Put/seqid=0 2024-12-06T06:16:26,459 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/A of b41c24c7a57a1a19a0dc6ea2edf56813 into 729cfb3dbfb8411ebd9120371ff3ccb6(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:26,459 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:26,459 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/A, priority=13, startTime=1733465786012; duration=0sec 2024-12-06T06:16:26,459 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:26,459 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:A 2024-12-06T06:16:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742405_1581 (size=12104) 2024-12-06T06:16:26,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742406_1582 (size=30955) 2024-12-06T06:16:26,489 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/d5ac6ae428e84941b28767c3a34f8374 2024-12-06T06:16:26,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/d10342bcb6ca451ab81c919e602be704 is 50, key is test_row_0/B:col10/1733465784783/Put/seqid=0 2024-12-06T06:16:26,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742407_1583 (size=12001) 2024-12-06T06:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T06:16:26,866 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/0f511a4927954126840e810acc72b64d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0f511a4927954126840e810acc72b64d 2024-12-06T06:16:26,871 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/C of b41c24c7a57a1a19a0dc6ea2edf56813 into 0f511a4927954126840e810acc72b64d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:26,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:26,871 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/C, priority=13, startTime=1733465786012; duration=0sec 2024-12-06T06:16:26,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:26,871 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:C 2024-12-06T06:16:26,901 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/d10342bcb6ca451ab81c919e602be704 2024-12-06T06:16:26,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/813f460b6a8044b2b2c09785d3704985 is 50, key is test_row_0/C:col10/1733465784783/Put/seqid=0 2024-12-06T06:16:26,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742408_1584 (size=12001) 2024-12-06T06:16:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:26,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465846947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:26,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465846948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:26,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465846949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:26,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465846951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:26,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:26,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:26,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465847063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465847063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465847064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465847064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465847273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465847273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465847274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465847274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,312 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/813f460b6a8044b2b2c09785d3704985 2024-12-06T06:16:27,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/d5ac6ae428e84941b28767c3a34f8374 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/d5ac6ae428e84941b28767c3a34f8374 2024-12-06T06:16:27,320 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/d5ac6ae428e84941b28767c3a34f8374, entries=150, sequenceid=82, filesize=30.2 K 2024-12-06T06:16:27,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/d10342bcb6ca451ab81c919e602be704 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d10342bcb6ca451ab81c919e602be704 2024-12-06T06:16:27,324 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d10342bcb6ca451ab81c919e602be704, entries=150, sequenceid=82, filesize=11.7 K 2024-12-06T06:16:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/813f460b6a8044b2b2c09785d3704985 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/813f460b6a8044b2b2c09785d3704985 2024-12-06T06:16:27,329 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/813f460b6a8044b2b2c09785d3704985, entries=150, sequenceid=82, filesize=11.7 K 2024-12-06T06:16:27,330 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1304ms, sequenceid=82, compaction requested=false 2024-12-06T06:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-06T06:16:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-06T06:16:27,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-06T06:16:27,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7640 sec 2024-12-06T06:16:27,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7690 sec 2024-12-06T06:16:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:27,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:16:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:27,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120623fb80aeb31b4b1592d33965b7c6480b_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465787585/Put/seqid=0 2024-12-06T06:16:27,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742409_1585 (size=12154) 2024-12-06T06:16:27,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465847636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465847641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465847642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465847642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T06:16:27,714 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:27,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-06T06:16:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T06:16:27,716 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:27,716 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:27,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:27,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465847754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465847754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465847754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465847754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T06:16:27,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:27,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:27,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465847964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465847964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465847964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465847964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,006 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:28,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120623fb80aeb31b4b1592d33965b7c6480b_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120623fb80aeb31b4b1592d33965b7c6480b_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:28,014 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/5fd54b9c9f054604a529b6eb4f54434b, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:28,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/5fd54b9c9f054604a529b6eb4f54434b is 175, key is test_row_0/A:col10/1733465787585/Put/seqid=0 2024-12-06T06:16:28,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742410_1586 (size=30955) 2024-12-06T06:16:28,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:28,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:28,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T06:16:28,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:28,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:28,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465848274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465848274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465848274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465848274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:28,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:28,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T06:16:28,419 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/5fd54b9c9f054604a529b6eb4f54434b 2024-12-06T06:16:28,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/5c7d35356e314e1a95545b391b9e1afc is 50, key is test_row_0/B:col10/1733465787585/Put/seqid=0 2024-12-06T06:16:28,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742411_1587 (size=12001) 2024-12-06T06:16:28,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:28,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:28,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:28,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:28,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:28,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465848784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465848784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465848784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465848784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:28,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/5c7d35356e314e1a95545b391b9e1afc 2024-12-06T06:16:28,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/b0ebc092d1d4402889c1a498b9dc5247 is 50, key is test_row_0/C:col10/1733465787585/Put/seqid=0 2024-12-06T06:16:28,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742412_1588 (size=12001) 2024-12-06T06:16:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T06:16:28,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:28,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:29,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:29,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/b0ebc092d1d4402889c1a498b9dc5247 2024-12-06T06:16:29,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/5fd54b9c9f054604a529b6eb4f54434b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/5fd54b9c9f054604a529b6eb4f54434b 2024-12-06T06:16:29,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:29,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/5fd54b9c9f054604a529b6eb4f54434b, entries=150, sequenceid=99, filesize=30.2 K 2024-12-06T06:16:29,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/5c7d35356e314e1a95545b391b9e1afc as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5c7d35356e314e1a95545b391b9e1afc 2024-12-06T06:16:29,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5c7d35356e314e1a95545b391b9e1afc, entries=150, sequenceid=99, filesize=11.7 K 2024-12-06T06:16:29,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/b0ebc092d1d4402889c1a498b9dc5247 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0ebc092d1d4402889c1a498b9dc5247 2024-12-06T06:16:29,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0ebc092d1d4402889c1a498b9dc5247, entries=150, sequenceid=99, filesize=11.7 K 2024-12-06T06:16:29,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1668ms, sequenceid=99, compaction requested=true 2024-12-06T06:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:29,255 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:29,255 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:29,256 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:29,256 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:29,256 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/A is initiating minor compaction (all files) 2024-12-06T06:16:29,256 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/B is initiating minor compaction (all files) 2024-12-06T06:16:29,256 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/B in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,256 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/A in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,256 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5ab66167b95e4ef39968598bfa6e5db6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d10342bcb6ca451ab81c919e602be704, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5c7d35356e314e1a95545b391b9e1afc] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=35.3 K 2024-12-06T06:16:29,256 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/729cfb3dbfb8411ebd9120371ff3ccb6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/d5ac6ae428e84941b28767c3a34f8374, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/5fd54b9c9f054604a529b6eb4f54434b] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=90.8 K 2024-12-06T06:16:29,256 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,256 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/729cfb3dbfb8411ebd9120371ff3ccb6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/d5ac6ae428e84941b28767c3a34f8374, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/5fd54b9c9f054604a529b6eb4f54434b] 2024-12-06T06:16:29,256 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab66167b95e4ef39968598bfa6e5db6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733465782607 2024-12-06T06:16:29,257 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 729cfb3dbfb8411ebd9120371ff3ccb6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733465782607 2024-12-06T06:16:29,257 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d10342bcb6ca451ab81c919e602be704, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733465784771 2024-12-06T06:16:29,257 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5ac6ae428e84941b28767c3a34f8374, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733465784771 2024-12-06T06:16:29,257 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fd54b9c9f054604a529b6eb4f54434b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465786947 2024-12-06T06:16:29,257 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c7d35356e314e1a95545b391b9e1afc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465786947 2024-12-06T06:16:29,263 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:29,263 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/24032255b0ad470384c7761c773deb7e is 50, key is test_row_0/B:col10/1733465787585/Put/seqid=0 2024-12-06T06:16:29,269 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:29,271 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412065232666f15ff4fd596ae64d819bd35f2_b41c24c7a57a1a19a0dc6ea2edf56813 store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:29,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742413_1589 (size=12207) 2024-12-06T06:16:29,273 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412065232666f15ff4fd596ae64d819bd35f2_b41c24c7a57a1a19a0dc6ea2edf56813, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:29,274 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065232666f15ff4fd596ae64d819bd35f2_b41c24c7a57a1a19a0dc6ea2edf56813 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:29,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742414_1590 (size=4469) 2024-12-06T06:16:29,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T06:16:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,398 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:16:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:29,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:29,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206248c45af866d46b28144eec123f72d84_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465787640/Put/seqid=0 2024-12-06T06:16:29,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742415_1591 (size=12154) 2024-12-06T06:16:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:29,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:29,679 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/24032255b0ad470384c7761c773deb7e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/24032255b0ad470384c7761c773deb7e 2024-12-06T06:16:29,680 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:29,680 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/7200d2c0ed0e4e8dbb21e8e54b5f53ea is 175, key is test_row_0/A:col10/1733465787585/Put/seqid=0 2024-12-06T06:16:29,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742416_1592 (size=31161) 2024-12-06T06:16:29,684 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/B of b41c24c7a57a1a19a0dc6ea2edf56813 into 24032255b0ad470384c7761c773deb7e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:29,684 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:29,684 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/B, priority=13, startTime=1733465789255; duration=0sec 2024-12-06T06:16:29,684 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:29,684 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:B 2024-12-06T06:16:29,684 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:29,686 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:29,686 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/C is initiating minor compaction (all files) 2024-12-06T06:16:29,686 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/C in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:29,687 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0f511a4927954126840e810acc72b64d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/813f460b6a8044b2b2c09785d3704985, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0ebc092d1d4402889c1a498b9dc5247] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=35.3 K 2024-12-06T06:16:29,687 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f511a4927954126840e810acc72b64d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733465782607 2024-12-06T06:16:29,687 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 813f460b6a8044b2b2c09785d3704985, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733465784771 2024-12-06T06:16:29,688 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ebc092d1d4402889c1a498b9dc5247, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465786947 2024-12-06T06:16:29,688 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/7200d2c0ed0e4e8dbb21e8e54b5f53ea as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7200d2c0ed0e4e8dbb21e8e54b5f53ea 2024-12-06T06:16:29,693 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/A of b41c24c7a57a1a19a0dc6ea2edf56813 into 7200d2c0ed0e4e8dbb21e8e54b5f53ea(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:29,693 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:29,693 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/A, priority=13, startTime=1733465789255; duration=0sec 2024-12-06T06:16:29,693 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:29,693 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:A 2024-12-06T06:16:29,694 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:29,695 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/9294085e68d54256800d82e79ade5f72 is 50, key is test_row_0/C:col10/1733465787585/Put/seqid=0 2024-12-06T06:16:29,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742417_1593 (size=12207) 2024-12-06T06:16:29,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465849731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465849794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465849794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465849794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:29,798 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465849794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,798 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:29,798 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:29,799 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_0' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:29,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:29,811 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206248c45af866d46b28144eec123f72d84_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206248c45af866d46b28144eec123f72d84_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:29,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/7346ec26460d45f3871fb62da1e7b5cb, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:29,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/7346ec26460d45f3871fb62da1e7b5cb is 175, key is test_row_0/A:col10/1733465787640/Put/seqid=0 2024-12-06T06:16:29,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742418_1594 (size=30955) 2024-12-06T06:16:29,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465849844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:29,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T06:16:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465850053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:30,102 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/9294085e68d54256800d82e79ade5f72 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/9294085e68d54256800d82e79ade5f72 2024-12-06T06:16:30,105 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/C of b41c24c7a57a1a19a0dc6ea2edf56813 into 9294085e68d54256800d82e79ade5f72(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:30,105 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:30,105 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/C, priority=13, startTime=1733465789255; duration=0sec 2024-12-06T06:16:30,105 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:30,105 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:C 2024-12-06T06:16:30,217 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/7346ec26460d45f3871fb62da1e7b5cb 2024-12-06T06:16:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/54aefb14d9e44a4084dff1530088c246 is 50, key is test_row_0/B:col10/1733465787640/Put/seqid=0 2024-12-06T06:16:30,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742419_1595 (size=12001) 2024-12-06T06:16:30,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465850364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:30,627 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/54aefb14d9e44a4084dff1530088c246 2024-12-06T06:16:30,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/082a5298fb034a0fbe9cf0c1d246b511 is 50, key is test_row_0/C:col10/1733465787640/Put/seqid=0 2024-12-06T06:16:30,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742420_1596 (size=12001) 2024-12-06T06:16:30,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465850873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,045 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/082a5298fb034a0fbe9cf0c1d246b511 2024-12-06T06:16:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/7346ec26460d45f3871fb62da1e7b5cb as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7346ec26460d45f3871fb62da1e7b5cb 2024-12-06T06:16:31,052 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7346ec26460d45f3871fb62da1e7b5cb, entries=150, sequenceid=117, filesize=30.2 K 2024-12-06T06:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/54aefb14d9e44a4084dff1530088c246 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/54aefb14d9e44a4084dff1530088c246 2024-12-06T06:16:31,056 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/54aefb14d9e44a4084dff1530088c246, entries=150, sequenceid=117, filesize=11.7 K 2024-12-06T06:16:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/082a5298fb034a0fbe9cf0c1d246b511 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/082a5298fb034a0fbe9cf0c1d246b511 2024-12-06T06:16:31,060 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/082a5298fb034a0fbe9cf0c1d246b511, entries=150, sequenceid=117, filesize=11.7 K 2024-12-06T06:16:31,061 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1663ms, sequenceid=117, compaction requested=false 2024-12-06T06:16:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-06T06:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-06T06:16:31,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-06T06:16:31,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3460 sec 2024-12-06T06:16:31,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.3510 sec 2024-12-06T06:16:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:31,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:16:31,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:31,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:31,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066d809d4b01d04aecadd714800bcffa54_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465791806/Put/seqid=0 2024-12-06T06:16:31,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742421_1597 (size=14794) 2024-12-06T06:16:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465851837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465851838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465851840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465851840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T06:16:31,884 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:31,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-06T06:16:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T06:16:31,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465851883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,887 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:31,887 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_2' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:31,887 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465851954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465851954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465851954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465851954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T06:16:32,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465852163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465852163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465852163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465852163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:32,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:32,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T06:16:32,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:32,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066d809d4b01d04aecadd714800bcffa54_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066d809d4b01d04aecadd714800bcffa54_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:32,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/fb2859b7baa04b598b477a020e6fe537, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:32,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/fb2859b7baa04b598b477a020e6fe537 is 175, key is test_row_0/A:col10/1733465791806/Put/seqid=0 2024-12-06T06:16:32,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742422_1598 (size=39749) 2024-12-06T06:16:32,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:32,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465852474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465852474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465852474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465852474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:32,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:32,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T06:16:32,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/fb2859b7baa04b598b477a020e6fe537 2024-12-06T06:16:32,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/6d3c52b98f274dc38ca35b9e2520f0e4 is 50, key is test_row_0/B:col10/1733465791806/Put/seqid=0 2024-12-06T06:16:32,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742423_1599 (size=12151) 2024-12-06T06:16:32,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:32,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:32,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:32,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:32,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:32,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:32,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:32,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465852984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465852984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465852984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:32,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465852984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T06:16:33,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/6d3c52b98f274dc38ca35b9e2520f0e4 2024-12-06T06:16:33,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/b0729fb3118d4942b6396ea0c52c5632 is 50, key is test_row_0/C:col10/1733465791806/Put/seqid=0 2024-12-06T06:16:33,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742424_1600 (size=12151) 2024-12-06T06:16:33,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:33,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:33,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:33,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:33,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:33,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:33,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:33,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/b0729fb3118d4942b6396ea0c52c5632 2024-12-06T06:16:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/fb2859b7baa04b598b477a020e6fe537 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/fb2859b7baa04b598b477a020e6fe537 2024-12-06T06:16:33,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/fb2859b7baa04b598b477a020e6fe537, entries=200, sequenceid=135, filesize=38.8 K 2024-12-06T06:16:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/6d3c52b98f274dc38ca35b9e2520f0e4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/6d3c52b98f274dc38ca35b9e2520f0e4 2024-12-06T06:16:33,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/6d3c52b98f274dc38ca35b9e2520f0e4, entries=150, sequenceid=135, filesize=11.9 K 2024-12-06T06:16:33,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/b0729fb3118d4942b6396ea0c52c5632 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0729fb3118d4942b6396ea0c52c5632 2024-12-06T06:16:33,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0729fb3118d4942b6396ea0c52c5632, entries=150, sequenceid=135, filesize=11.9 K 2024-12-06T06:16:33,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1676ms, sequenceid=135, compaction requested=true 2024-12-06T06:16:33,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:33,484 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:33,485 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:33,485 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/A is initiating minor compaction (all files) 2024-12-06T06:16:33,485 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/A in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,485 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7200d2c0ed0e4e8dbb21e8e54b5f53ea, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7346ec26460d45f3871fb62da1e7b5cb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/fb2859b7baa04b598b477a020e6fe537] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=99.5 K 2024-12-06T06:16:33,485 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,485 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7200d2c0ed0e4e8dbb21e8e54b5f53ea, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7346ec26460d45f3871fb62da1e7b5cb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/fb2859b7baa04b598b477a020e6fe537] 2024-12-06T06:16:33,486 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7200d2c0ed0e4e8dbb21e8e54b5f53ea, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465786947 2024-12-06T06:16:33,487 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7346ec26460d45f3871fb62da1e7b5cb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733465787634 2024-12-06T06:16:33,487 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb2859b7baa04b598b477a020e6fe537, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733465789719 2024-12-06T06:16:33,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:33,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:33,491 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:33,492 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:33,492 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/B is initiating minor compaction (all files) 2024-12-06T06:16:33,492 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/B in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,492 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/24032255b0ad470384c7761c773deb7e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/54aefb14d9e44a4084dff1530088c246, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/6d3c52b98f274dc38ca35b9e2520f0e4] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=35.5 K 2024-12-06T06:16:33,492 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 24032255b0ad470384c7761c773deb7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465786947 2024-12-06T06:16:33,493 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 54aefb14d9e44a4084dff1530088c246, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733465787634 2024-12-06T06:16:33,493 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d3c52b98f274dc38ca35b9e2520f0e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733465789719 2024-12-06T06:16:33,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:33,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:33,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:33,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:33,498 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:33,515 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:33,515 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/e6e7749b61bf456c8b53f13cce1f2eee is 50, key is test_row_0/B:col10/1733465791806/Put/seqid=0 2024-12-06T06:16:33,529 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120609b0f5ddb6da412b9f08a41548ec459b_b41c24c7a57a1a19a0dc6ea2edf56813 store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:33,531 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120609b0f5ddb6da412b9f08a41548ec459b_b41c24c7a57a1a19a0dc6ea2edf56813, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:33,531 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120609b0f5ddb6da412b9f08a41548ec459b_b41c24c7a57a1a19a0dc6ea2edf56813 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:33,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742425_1601 (size=12459) 2024-12-06T06:16:33,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T06:16:33,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,571 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:16:33,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:33,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:33,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:33,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:33,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:33,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:33,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742426_1602 (size=4469) 2024-12-06T06:16:33,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ff7347b8c6384a03a90735a300bd150a_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465791836/Put/seqid=0 2024-12-06T06:16:33,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742427_1603 (size=12304) 2024-12-06T06:16:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:33,641 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ff7347b8c6384a03a90735a300bd150a_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ff7347b8c6384a03a90735a300bd150a_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:33,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/e1f1e970a0c543c9a8552f24fb17ef96, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:33,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/e1f1e970a0c543c9a8552f24fb17ef96 is 175, key is test_row_0/A:col10/1733465791836/Put/seqid=0 2024-12-06T06:16:33,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742428_1604 (size=31105) 2024-12-06T06:16:33,666 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/e1f1e970a0c543c9a8552f24fb17ef96 2024-12-06T06:16:33,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/97f53005c2e74ee8b572303bb324473c is 50, key is test_row_0/B:col10/1733465791836/Put/seqid=0 2024-12-06T06:16:33,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742429_1605 (size=12151) 2024-12-06T06:16:33,697 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/97f53005c2e74ee8b572303bb324473c 2024-12-06T06:16:33,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/c3c123308c244139812e2951cd2dd662 is 50, key is test_row_0/C:col10/1733465791836/Put/seqid=0 2024-12-06T06:16:33,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742430_1606 (size=12151) 2024-12-06T06:16:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:33,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465853960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:33,972 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/e6e7749b61bf456c8b53f13cce1f2eee as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/e6e7749b61bf456c8b53f13cce1f2eee 2024-12-06T06:16:33,976 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/B of b41c24c7a57a1a19a0dc6ea2edf56813 into e6e7749b61bf456c8b53f13cce1f2eee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:33,976 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:33,976 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/B, priority=13, startTime=1733465793491; duration=0sec 2024-12-06T06:16:33,977 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:33,977 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:B 2024-12-06T06:16:33,977 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:33,977 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:33,977 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/C is initiating minor compaction (all files) 2024-12-06T06:16:33,977 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/C in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:33,978 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/9294085e68d54256800d82e79ade5f72, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/082a5298fb034a0fbe9cf0c1d246b511, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0729fb3118d4942b6396ea0c52c5632] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=35.5 K 2024-12-06T06:16:33,978 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9294085e68d54256800d82e79ade5f72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733465786947 2024-12-06T06:16:33,978 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 082a5298fb034a0fbe9cf0c1d246b511, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733465787634 2024-12-06T06:16:33,978 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting b0729fb3118d4942b6396ea0c52c5632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733465789719 2024-12-06T06:16:33,984 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:33,985 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/8addfbcb23554a70a686b791e15b2477 is 50, key is test_row_0/C:col10/1733465791806/Put/seqid=0 2024-12-06T06:16:33,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742431_1607 (size=12459) 2024-12-06T06:16:34,001 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#A#compaction#512 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:34,002 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/3c495fbf5c51452e967fae01b269059b is 175, key is test_row_0/A:col10/1733465791806/Put/seqid=0 2024-12-06T06:16:34,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742432_1608 (size=31413) 2024-12-06T06:16:34,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465854003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465854003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465854003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,007 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:34,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,007 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465854004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:34,007 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:34,008 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T06:16:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465854073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:34,116 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/c3c123308c244139812e2951cd2dd662 2024-12-06T06:16:34,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/e1f1e970a0c543c9a8552f24fb17ef96 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1f1e970a0c543c9a8552f24fb17ef96 2024-12-06T06:16:34,123 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1f1e970a0c543c9a8552f24fb17ef96, entries=150, sequenceid=153, filesize=30.4 K 2024-12-06T06:16:34,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/97f53005c2e74ee8b572303bb324473c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/97f53005c2e74ee8b572303bb324473c 2024-12-06T06:16:34,127 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/97f53005c2e74ee8b572303bb324473c, entries=150, sequenceid=153, filesize=11.9 K 2024-12-06T06:16:34,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/c3c123308c244139812e2951cd2dd662 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/c3c123308c244139812e2951cd2dd662 2024-12-06T06:16:34,130 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/c3c123308c244139812e2951cd2dd662, entries=150, sequenceid=153, filesize=11.9 K 2024-12-06T06:16:34,131 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b41c24c7a57a1a19a0dc6ea2edf56813 in 561ms, sequenceid=153, compaction requested=false 2024-12-06T06:16:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-06T06:16:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-06T06:16:34,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-06T06:16:34,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2450 sec 2024-12-06T06:16:34,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.2510 sec 2024-12-06T06:16:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:34,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:16:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:34,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ced23571996f49caa935dc09d4a9769d_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465793945/Put/seqid=0 2024-12-06T06:16:34,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742433_1609 (size=14794) 2024-12-06T06:16:34,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465854360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:34,396 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/8addfbcb23554a70a686b791e15b2477 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8addfbcb23554a70a686b791e15b2477 2024-12-06T06:16:34,400 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/C of b41c24c7a57a1a19a0dc6ea2edf56813 into 8addfbcb23554a70a686b791e15b2477(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:34,400 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:34,401 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/C, priority=13, startTime=1733465793498; duration=0sec 2024-12-06T06:16:34,401 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:34,401 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:C 2024-12-06T06:16:34,410 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/3c495fbf5c51452e967fae01b269059b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3c495fbf5c51452e967fae01b269059b 2024-12-06T06:16:34,414 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/A of b41c24c7a57a1a19a0dc6ea2edf56813 into 3c495fbf5c51452e967fae01b269059b(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:34,414 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:34,414 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/A, priority=13, startTime=1733465793484; duration=0sec 2024-12-06T06:16:34,414 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:34,414 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:A 2024-12-06T06:16:34,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465854473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:34,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465854684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:34,698 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:34,702 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ced23571996f49caa935dc09d4a9769d_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ced23571996f49caa935dc09d4a9769d_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:34,704 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/54ab6649cb004394a40ae59680d7125b, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:34,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/54ab6649cb004394a40ae59680d7125b is 175, key is test_row_0/A:col10/1733465793945/Put/seqid=0 2024-12-06T06:16:34,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742434_1610 (size=39749) 2024-12-06T06:16:34,711 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/54ab6649cb004394a40ae59680d7125b 2024-12-06T06:16:34,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/77ada57b72324ab8b0c6fa3cbd7ec4a7 is 50, key is test_row_0/B:col10/1733465793945/Put/seqid=0 2024-12-06T06:16:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742435_1611 (size=12151) 2024-12-06T06:16:34,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/77ada57b72324ab8b0c6fa3cbd7ec4a7 2024-12-06T06:16:34,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/f2716104045149c997ef39fbf840bbda is 50, key is test_row_0/C:col10/1733465793945/Put/seqid=0 2024-12-06T06:16:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742436_1612 (size=12151) 2024-12-06T06:16:34,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465854993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:35,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/f2716104045149c997ef39fbf840bbda 2024-12-06T06:16:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/54ab6649cb004394a40ae59680d7125b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/54ab6649cb004394a40ae59680d7125b 2024-12-06T06:16:35,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/54ab6649cb004394a40ae59680d7125b, entries=200, sequenceid=169, filesize=38.8 K 2024-12-06T06:16:35,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/77ada57b72324ab8b0c6fa3cbd7ec4a7 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/77ada57b72324ab8b0c6fa3cbd7ec4a7 2024-12-06T06:16:35,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/77ada57b72324ab8b0c6fa3cbd7ec4a7, entries=150, sequenceid=169, filesize=11.9 K 2024-12-06T06:16:35,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/f2716104045149c997ef39fbf840bbda as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/f2716104045149c997ef39fbf840bbda 2024-12-06T06:16:35,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/f2716104045149c997ef39fbf840bbda, entries=150, sequenceid=169, filesize=11.9 K 2024-12-06T06:16:35,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b41c24c7a57a1a19a0dc6ea2edf56813 in 870ms, sequenceid=169, compaction requested=true 2024-12-06T06:16:35,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:35,158 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T06:16:35,158 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:35,160 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:35,160 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:35,161 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/B is initiating minor compaction (all files) 2024-12-06T06:16:35,161 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/A is initiating minor compaction (all files) 2024-12-06T06:16:35,161 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/A in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:35,161 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/B in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:35,161 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3c495fbf5c51452e967fae01b269059b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1f1e970a0c543c9a8552f24fb17ef96, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/54ab6649cb004394a40ae59680d7125b] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=99.9 K 2024-12-06T06:16:35,161 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:35,161 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/e6e7749b61bf456c8b53f13cce1f2eee, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/97f53005c2e74ee8b572303bb324473c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/77ada57b72324ab8b0c6fa3cbd7ec4a7] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=35.9 K 2024-12-06T06:16:35,161 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3c495fbf5c51452e967fae01b269059b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1f1e970a0c543c9a8552f24fb17ef96, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/54ab6649cb004394a40ae59680d7125b] 2024-12-06T06:16:35,162 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e7749b61bf456c8b53f13cce1f2eee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733465789719 2024-12-06T06:16:35,162 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c495fbf5c51452e967fae01b269059b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733465789719 2024-12-06T06:16:35,162 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f1e970a0c543c9a8552f24fb17ef96, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733465791836 2024-12-06T06:16:35,163 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f53005c2e74ee8b572303bb324473c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733465791836 2024-12-06T06:16:35,163 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54ab6649cb004394a40ae59680d7125b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465793939 2024-12-06T06:16:35,164 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ada57b72324ab8b0c6fa3cbd7ec4a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465793945 2024-12-06T06:16:35,171 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:35,171 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/a1571d7694634fbcb1fc03cbae26f03b is 50, key is test_row_0/B:col10/1733465793945/Put/seqid=0 2024-12-06T06:16:35,172 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:35,174 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120676e67fcfd62440129539a4223c3868f2_b41c24c7a57a1a19a0dc6ea2edf56813 store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:35,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742437_1613 (size=12561) 2024-12-06T06:16:35,176 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120676e67fcfd62440129539a4223c3868f2_b41c24c7a57a1a19a0dc6ea2edf56813, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:35,176 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120676e67fcfd62440129539a4223c3868f2_b41c24c7a57a1a19a0dc6ea2edf56813 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:35,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742438_1614 (size=4469) 2024-12-06T06:16:35,180 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#A#compaction#522 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:35,181 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/3a5ec64cc5e446a681ecc93cc5544af3 is 175, key is test_row_0/A:col10/1733465793945/Put/seqid=0 2024-12-06T06:16:35,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742439_1615 (size=31515) 2024-12-06T06:16:35,189 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/3a5ec64cc5e446a681ecc93cc5544af3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3a5ec64cc5e446a681ecc93cc5544af3 2024-12-06T06:16:35,193 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/A of b41c24c7a57a1a19a0dc6ea2edf56813 into 3a5ec64cc5e446a681ecc93cc5544af3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:35,193 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:35,193 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/A, priority=13, startTime=1733465795157; duration=0sec 2024-12-06T06:16:35,193 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:35,193 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:A 2024-12-06T06:16:35,193 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:35,194 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:35,194 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/C is initiating minor compaction (all files) 2024-12-06T06:16:35,194 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/C in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:35,194 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8addfbcb23554a70a686b791e15b2477, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/c3c123308c244139812e2951cd2dd662, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/f2716104045149c997ef39fbf840bbda] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=35.9 K 2024-12-06T06:16:35,195 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8addfbcb23554a70a686b791e15b2477, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733465789719 2024-12-06T06:16:35,195 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c123308c244139812e2951cd2dd662, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733465791836 2024-12-06T06:16:35,195 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2716104045149c997ef39fbf840bbda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465793945 2024-12-06T06:16:35,200 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#C#compaction#523 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:35,201 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/59b7862f14194142a5beb7aa9f69158d is 50, key is test_row_0/C:col10/1733465793945/Put/seqid=0 2024-12-06T06:16:35,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742440_1616 (size=12561) 2024-12-06T06:16:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:35,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:16:35,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:35,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:35,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:35,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:35,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:35,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:35,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206259c362f782d4706b645461021c747ac_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465795506/Put/seqid=0 2024-12-06T06:16:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742441_1617 (size=12304) 2024-12-06T06:16:35,522 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:35,526 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206259c362f782d4706b645461021c747ac_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206259c362f782d4706b645461021c747ac_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:35,527 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/1a196e4d18784815ab458147a670ab74, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:35,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/1a196e4d18784815ab458147a670ab74 is 175, key is test_row_0/A:col10/1733465795506/Put/seqid=0 2024-12-06T06:16:35,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742442_1618 (size=31105) 2024-12-06T06:16:35,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/1a196e4d18784815ab458147a670ab74 2024-12-06T06:16:35,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/563d96e490d84878b829af364bf04421 is 50, key is test_row_0/B:col10/1733465795506/Put/seqid=0 2024-12-06T06:16:35,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742443_1619 (size=12151) 2024-12-06T06:16:35,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465855568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:35,579 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/a1571d7694634fbcb1fc03cbae26f03b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/a1571d7694634fbcb1fc03cbae26f03b 2024-12-06T06:16:35,587 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/B of b41c24c7a57a1a19a0dc6ea2edf56813 into a1571d7694634fbcb1fc03cbae26f03b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:35,587 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:35,587 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/B, priority=13, startTime=1733465795158; duration=0sec 2024-12-06T06:16:35,587 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:35,587 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:B 2024-12-06T06:16:35,608 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/59b7862f14194142a5beb7aa9f69158d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/59b7862f14194142a5beb7aa9f69158d 2024-12-06T06:16:35,612 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/C of b41c24c7a57a1a19a0dc6ea2edf56813 into 59b7862f14194142a5beb7aa9f69158d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:35,612 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:35,612 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/C, priority=13, startTime=1733465795158; duration=0sec 2024-12-06T06:16:35,612 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:35,612 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:C 2024-12-06T06:16:35,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465855684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:35,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465855893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:35,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:35,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:35,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:35,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/563d96e490d84878b829af364bf04421 2024-12-06T06:16:35,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/5f0eb8edd20f41fe924dcc618de6e145 is 50, key is test_row_0/C:col10/1733465795506/Put/seqid=0 2024-12-06T06:16:35,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742444_1620 (size=12151) 2024-12-06T06:16:35,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/5f0eb8edd20f41fe924dcc618de6e145 2024-12-06T06:16:35,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/1a196e4d18784815ab458147a670ab74 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/1a196e4d18784815ab458147a670ab74 2024-12-06T06:16:35,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/1a196e4d18784815ab458147a670ab74, entries=150, sequenceid=190, filesize=30.4 K 2024-12-06T06:16:35,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/563d96e490d84878b829af364bf04421 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/563d96e490d84878b829af364bf04421 2024-12-06T06:16:35,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/563d96e490d84878b829af364bf04421, entries=150, sequenceid=190, filesize=11.9 K 2024-12-06T06:16:35,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/5f0eb8edd20f41fe924dcc618de6e145 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/5f0eb8edd20f41fe924dcc618de6e145 2024-12-06T06:16:35,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/5f0eb8edd20f41fe924dcc618de6e145, entries=150, sequenceid=190, filesize=11.9 K 2024-12-06T06:16:35,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b41c24c7a57a1a19a0dc6ea2edf56813 in 473ms, sequenceid=190, compaction requested=false 2024-12-06T06:16:35,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:36,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:16:36,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:36,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:36,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:36,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:36,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:36,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:36,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ab0792204d134b5889b1661cecce156a_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465795555/Put/seqid=0 2024-12-06T06:16:36,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742445_1621 (size=14794) 2024-12-06T06:16:36,041 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:36,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ab0792204d134b5889b1661cecce156a_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ab0792204d134b5889b1661cecce156a_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:36,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/cb7c94107661487c92c7f06e37c67d6d, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:36,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/cb7c94107661487c92c7f06e37c67d6d is 175, key is test_row_0/A:col10/1733465795555/Put/seqid=0 2024-12-06T06:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T06:16:36,055 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:36,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742446_1622 (size=39749) 2024-12-06T06:16:36,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-06T06:16:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465856050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,057 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465856051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T06:16:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,058 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/cb7c94107661487c92c7f06e37c67d6d 2024-12-06T06:16:36,059 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:36,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:36,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465856058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465856059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/45f7258869f14c60afeb16ed28ea6612 is 50, key is test_row_0/B:col10/1733465795555/Put/seqid=0 2024-12-06T06:16:36,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742447_1623 (size=12151) 2024-12-06T06:16:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T06:16:36,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465856164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465856164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465856174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465856174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465856203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T06:16:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T06:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T06:16:36,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465856373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465856373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465856384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465856384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/45f7258869f14c60afeb16ed28ea6612 2024-12-06T06:16:36,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/3c8ab526755c4ae6a83aa127cbc81736 is 50, key is test_row_0/C:col10/1733465795555/Put/seqid=0 2024-12-06T06:16:36,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742448_1624 (size=12151) 2024-12-06T06:16:36,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/3c8ab526755c4ae6a83aa127cbc81736 2024-12-06T06:16:36,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/cb7c94107661487c92c7f06e37c67d6d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cb7c94107661487c92c7f06e37c67d6d 2024-12-06T06:16:36,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cb7c94107661487c92c7f06e37c67d6d, entries=200, sequenceid=207, filesize=38.8 K 2024-12-06T06:16:36,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/45f7258869f14c60afeb16ed28ea6612 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/45f7258869f14c60afeb16ed28ea6612 2024-12-06T06:16:36,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/45f7258869f14c60afeb16ed28ea6612, entries=150, sequenceid=207, filesize=11.9 K 2024-12-06T06:16:36,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/3c8ab526755c4ae6a83aa127cbc81736 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/3c8ab526755c4ae6a83aa127cbc81736 2024-12-06T06:16:36,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/3c8ab526755c4ae6a83aa127cbc81736, entries=150, sequenceid=207, filesize=11.9 K 2024-12-06T06:16:36,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T06:16:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b41c24c7a57a1a19a0dc6ea2edf56813 in 501ms, sequenceid=207, compaction requested=true 2024-12-06T06:16:36,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:36,519 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:36,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:36,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:36,520 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:36,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:36,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:36,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:36,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:36,529 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:36,529 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/A is initiating minor compaction (all files) 2024-12-06T06:16:36,529 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/A in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,529 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3a5ec64cc5e446a681ecc93cc5544af3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/1a196e4d18784815ab458147a670ab74, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cb7c94107661487c92c7f06e37c67d6d] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=100.0 K 2024-12-06T06:16:36,529 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,529 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3a5ec64cc5e446a681ecc93cc5544af3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/1a196e4d18784815ab458147a670ab74, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cb7c94107661487c92c7f06e37c67d6d] 2024-12-06T06:16:36,530 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:36,530 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a5ec64cc5e446a681ecc93cc5544af3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465793945 2024-12-06T06:16:36,530 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/B is initiating minor compaction (all files) 2024-12-06T06:16:36,530 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/B in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,530 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/a1571d7694634fbcb1fc03cbae26f03b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/563d96e490d84878b829af364bf04421, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/45f7258869f14c60afeb16ed28ea6612] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=36.0 K 2024-12-06T06:16:36,530 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a196e4d18784815ab458147a670ab74, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733465794323 2024-12-06T06:16:36,530 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting a1571d7694634fbcb1fc03cbae26f03b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465793945 2024-12-06T06:16:36,531 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb7c94107661487c92c7f06e37c67d6d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465795555 2024-12-06T06:16:36,531 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 563d96e490d84878b829af364bf04421, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733465794323 2024-12-06T06:16:36,532 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f7258869f14c60afeb16ed28ea6612, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465795555 2024-12-06T06:16:36,540 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:36,541 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412069c32151949864a758a04f5a5daf23877_b41c24c7a57a1a19a0dc6ea2edf56813 store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:36,542 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:36,542 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/3dcc1cea10744a139c9fddc858a41727 is 50, key is test_row_0/B:col10/1733465795555/Put/seqid=0 2024-12-06T06:16:36,543 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412069c32151949864a758a04f5a5daf23877_b41c24c7a57a1a19a0dc6ea2edf56813, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:36,544 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412069c32151949864a758a04f5a5daf23877_b41c24c7a57a1a19a0dc6ea2edf56813 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:36,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742449_1625 (size=12663) 2024-12-06T06:16:36,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742450_1626 (size=4469) 2024-12-06T06:16:36,565 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#A#compaction#530 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:36,565 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/77ffdca074784d9eb8427cafda7f30f2 is 175, key is test_row_0/A:col10/1733465795555/Put/seqid=0 2024-12-06T06:16:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742451_1627 (size=31617) 2024-12-06T06:16:36,580 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/77ffdca074784d9eb8427cafda7f30f2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/77ffdca074784d9eb8427cafda7f30f2 2024-12-06T06:16:36,586 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/A of b41c24c7a57a1a19a0dc6ea2edf56813 into 77ffdca074784d9eb8427cafda7f30f2(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:36,586 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:36,586 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/A, priority=13, startTime=1733465796519; duration=0sec 2024-12-06T06:16:36,586 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:36,586 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:A 2024-12-06T06:16:36,586 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T06:16:36,588 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T06:16:36,588 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/C is initiating minor compaction (all files) 2024-12-06T06:16:36,588 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/C in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,588 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/59b7862f14194142a5beb7aa9f69158d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/5f0eb8edd20f41fe924dcc618de6e145, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/3c8ab526755c4ae6a83aa127cbc81736] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=36.0 K 2024-12-06T06:16:36,589 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59b7862f14194142a5beb7aa9f69158d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733465793945 2024-12-06T06:16:36,590 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f0eb8edd20f41fe924dcc618de6e145, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733465794323 2024-12-06T06:16:36,590 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c8ab526755c4ae6a83aa127cbc81736, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465795555 2024-12-06T06:16:36,598 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#C#compaction#532 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:36,598 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/6ce6fbcff2404c1ea47fdcaf8339f81f is 50, key is test_row_0/C:col10/1733465795555/Put/seqid=0 2024-12-06T06:16:36,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742452_1628 (size=12663) 2024-12-06T06:16:36,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T06:16:36,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:36,672 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-06T06:16:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T06:16:36,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206165997eca1b945eea306cacbd7f21c88_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465796055/Put/seqid=0 2024-12-06T06:16:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:36,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:36,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742453_1629 (size=12304) 2024-12-06T06:16:36,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:36,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465856710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465856709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465856713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465856714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465856719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,731 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206165997eca1b945eea306cacbd7f21c88_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206165997eca1b945eea306cacbd7f21c88_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:36,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/cbcb394ac32c45d28bfdbe016827a496, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/cbcb394ac32c45d28bfdbe016827a496 is 175, key is test_row_0/A:col10/1733465796055/Put/seqid=0 2024-12-06T06:16:36,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742454_1630 (size=31105) 2024-12-06T06:16:36,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465856824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465856825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465856833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465856833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:36,966 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/3dcc1cea10744a139c9fddc858a41727 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/3dcc1cea10744a139c9fddc858a41727 2024-12-06T06:16:36,969 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/B of b41c24c7a57a1a19a0dc6ea2edf56813 into 3dcc1cea10744a139c9fddc858a41727(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:36,969 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:36,970 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/B, priority=13, startTime=1733465796519; duration=0sec 2024-12-06T06:16:36,970 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:36,970 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:B 2024-12-06T06:16:37,023 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/6ce6fbcff2404c1ea47fdcaf8339f81f as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6ce6fbcff2404c1ea47fdcaf8339f81f 2024-12-06T06:16:37,028 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/C of b41c24c7a57a1a19a0dc6ea2edf56813 into 6ce6fbcff2404c1ea47fdcaf8339f81f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:37,028 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:37,028 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/C, priority=13, startTime=1733465796520; duration=0sec 2024-12-06T06:16:37,028 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:37,028 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:C 2024-12-06T06:16:37,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465857034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465857034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465857044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465857044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,154 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/cbcb394ac32c45d28bfdbe016827a496 2024-12-06T06:16:37,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/126f85d0bb59428da041e6f019a783ab is 50, key is test_row_0/B:col10/1733465796055/Put/seqid=0 2024-12-06T06:16:37,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742455_1631 (size=12151) 2024-12-06T06:16:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T06:16:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465857344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465857344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465857354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465857354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,574 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/126f85d0bb59428da041e6f019a783ab 2024-12-06T06:16:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/971d2ac7728f4f9ab1ee44ad794f26f3 is 50, key is test_row_0/C:col10/1733465796055/Put/seqid=0 2024-12-06T06:16:37,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742456_1632 (size=12151) 2024-12-06T06:16:37,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465857734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,737 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465857854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465857854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465857863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465857863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:37,986 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/971d2ac7728f4f9ab1ee44ad794f26f3 2024-12-06T06:16:37,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/cbcb394ac32c45d28bfdbe016827a496 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cbcb394ac32c45d28bfdbe016827a496 2024-12-06T06:16:37,993 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cbcb394ac32c45d28bfdbe016827a496, entries=150, sequenceid=225, filesize=30.4 K 2024-12-06T06:16:37,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/126f85d0bb59428da041e6f019a783ab as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/126f85d0bb59428da041e6f019a783ab 2024-12-06T06:16:37,997 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/126f85d0bb59428da041e6f019a783ab, entries=150, sequenceid=225, filesize=11.9 K 2024-12-06T06:16:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/971d2ac7728f4f9ab1ee44ad794f26f3 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/971d2ac7728f4f9ab1ee44ad794f26f3 2024-12-06T06:16:38,000 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/971d2ac7728f4f9ab1ee44ad794f26f3, entries=150, sequenceid=225, filesize=11.9 K 2024-12-06T06:16:38,001 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1329ms, sequenceid=225, compaction requested=false 2024-12-06T06:16:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-06T06:16:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-06T06:16:38,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-06T06:16:38,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-12-06T06:16:38,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.9500 sec 2024-12-06T06:16:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T06:16:38,204 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:38,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-06T06:16:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T06:16:38,206 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:38,207 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:38,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T06:16:38,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T06:16:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:38,362 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-06T06:16:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:38,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:38,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:38,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:38,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:38,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:38,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206eeb540db78924281a7dbd92141ab0787_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465796716/Put/seqid=0 2024-12-06T06:16:38,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742457_1633 (size=12304) 2024-12-06T06:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T06:16:38,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:38,776 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206eeb540db78924281a7dbd92141ab0787_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eeb540db78924281a7dbd92141ab0787_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/18fa7cf2c9cd4d6494f579a98250a2e4, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/18fa7cf2c9cd4d6494f579a98250a2e4 is 175, key is test_row_0/A:col10/1733465796716/Put/seqid=0 2024-12-06T06:16:38,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742458_1634 (size=31105) 2024-12-06T06:16:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T06:16:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:38,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:38,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465858892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:38,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465858895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465858896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465858897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465859003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465859003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465859004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465859014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,181 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/18fa7cf2c9cd4d6494f579a98250a2e4 2024-12-06T06:16:39,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/1e0f8f84bb7f41ecbefc0b186c3bd07b is 50, key is test_row_0/B:col10/1733465796716/Put/seqid=0 2024-12-06T06:16:39,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742459_1635 (size=12151) 2024-12-06T06:16:39,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465859213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465859213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465859213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465859224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,291 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T06:16:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T06:16:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465859524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465859524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465859524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465859533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:39,591 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/1e0f8f84bb7f41ecbefc0b186c3bd07b 2024-12-06T06:16:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/8146ca4f18ad4f6b9a1658268905ee7a is 50, key is test_row_0/C:col10/1733465796716/Put/seqid=0 2024-12-06T06:16:39,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742460_1636 (size=12151) 2024-12-06T06:16:39,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39612 deadline: 1733465859744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:39,748 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6336e8474e1c,39987,1733465620995 for 'test_row_1' in b41c24c7a57a1a19a0dc6ea2edf56813 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-06T06:16:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,001 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/8146ca4f18ad4f6b9a1658268905ee7a 2024-12-06T06:16:40,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/18fa7cf2c9cd4d6494f579a98250a2e4 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/18fa7cf2c9cd4d6494f579a98250a2e4 2024-12-06T06:16:40,008 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/18fa7cf2c9cd4d6494f579a98250a2e4, entries=150, sequenceid=242, filesize=30.4 K 2024-12-06T06:16:40,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/1e0f8f84bb7f41ecbefc0b186c3bd07b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/1e0f8f84bb7f41ecbefc0b186c3bd07b 2024-12-06T06:16:40,012 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/1e0f8f84bb7f41ecbefc0b186c3bd07b, entries=150, sequenceid=242, filesize=11.9 K 2024-12-06T06:16:40,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/8146ca4f18ad4f6b9a1658268905ee7a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8146ca4f18ad4f6b9a1658268905ee7a 2024-12-06T06:16:40,016 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8146ca4f18ad4f6b9a1658268905ee7a, entries=150, sequenceid=242, filesize=11.9 K 2024-12-06T06:16:40,016 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1654ms, sequenceid=242, compaction requested=true 2024-12-06T06:16:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-06T06:16:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-06T06:16:40,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-06T06:16:40,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8110 sec 2024-12-06T06:16:40,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8160 sec 2024-12-06T06:16:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:40,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-06T06:16:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:40,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ed12c0be46ce480a9e63043c529af290_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465800037/Put/seqid=0 2024-12-06T06:16:40,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742461_1637 (size=14944) 2024-12-06T06:16:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465860051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465860051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465860052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465860056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465860164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465860164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465860164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465860164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T06:16:40,354 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:40,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T06:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-06T06:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T06:16:40,356 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T06:16:40,356 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T06:16:40,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T06:16:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465860374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465860374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465860374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465860374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,448 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:40,451 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ed12c0be46ce480a9e63043c529af290_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ed12c0be46ce480a9e63043c529af290_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:40,452 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/8bcb3c40415344eaa1e004a10aeccafe, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:40,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/8bcb3c40415344eaa1e004a10aeccafe is 175, key is test_row_0/A:col10/1733465800037/Put/seqid=0 2024-12-06T06:16:40,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742462_1638 (size=39899) 2024-12-06T06:16:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T06:16:40,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T06:16:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465860684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465860684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465860684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465860684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:40,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,857 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/8bcb3c40415344eaa1e004a10aeccafe 2024-12-06T06:16:40,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/d54a2ec3cf6441aabac7b9efc6024c6a is 50, key is test_row_0/B:col10/1733465800037/Put/seqid=0 2024-12-06T06:16:40,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742463_1639 (size=12251) 2024-12-06T06:16:40,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:40,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T06:16:41,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733465861194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:41,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39624 deadline: 1733465861194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:41,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733465861194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:41,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T06:16:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1733465861194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 , the old value is region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T06:16:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., hostname=6336e8474e1c,39987,1733465620995, seqNum=5 because the exception is null or not the one we care about 2024-12-06T06:16:41,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/d54a2ec3cf6441aabac7b9efc6024c6a 2024-12-06T06:16:41,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/147e62ff9b564982b53155d1139c4a7b is 50, key is test_row_0/C:col10/1733465800037/Put/seqid=0 2024-12-06T06:16:41,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:41,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:41,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742464_1640 (size=12251) 2024-12-06T06:16:41,401 INFO [Thread-2526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2526. 2024-12-06T06:16:41,401 INFO [Thread-2518 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2518. 2024-12-06T06:16:41,401 DEBUG [Thread-2526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:41,401 DEBUG [Thread-2518 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:41,401 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,401 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,401 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,401 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:41,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:41,405 INFO [Thread-2524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2524. 2024-12-06T06:16:41,405 DEBUG [Thread-2524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:41,405 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,405 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:41,405 INFO [Thread-2520 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2520. 2024-12-06T06:16:41,405 DEBUG [Thread-2520 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:41,405 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,406 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:41,406 INFO [Thread-2522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2522. 2024-12-06T06:16:41,406 DEBUG [Thread-2522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:41,406 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,406 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:41,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:41,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:41,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:41,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T06:16:41,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T06:16:41,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/147e62ff9b564982b53155d1139c4a7b 2024-12-06T06:16:41,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/8bcb3c40415344eaa1e004a10aeccafe as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/8bcb3c40415344eaa1e004a10aeccafe 2024-12-06T06:16:41,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/8bcb3c40415344eaa1e004a10aeccafe, entries=200, sequenceid=261, filesize=39.0 K 2024-12-06T06:16:41,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/d54a2ec3cf6441aabac7b9efc6024c6a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d54a2ec3cf6441aabac7b9efc6024c6a 2024-12-06T06:16:41,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d54a2ec3cf6441aabac7b9efc6024c6a, entries=150, sequenceid=261, filesize=12.0 K 2024-12-06T06:16:41,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/147e62ff9b564982b53155d1139c4a7b as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/147e62ff9b564982b53155d1139c4a7b 2024-12-06T06:16:41,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/147e62ff9b564982b53155d1139c4a7b, entries=150, sequenceid=261, filesize=12.0 K 2024-12-06T06:16:41,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1651ms, sequenceid=261, compaction requested=true 2024-12-06T06:16:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T06:16:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T06:16:41,689 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b41c24c7a57a1a19a0dc6ea2edf56813:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T06:16:41,689 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:41,690 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:41,690 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49216 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:41,690 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/B is initiating minor compaction (all files) 2024-12-06T06:16:41,690 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/A is initiating minor compaction (all files) 2024-12-06T06:16:41,690 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/A in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,690 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/B in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,690 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/3dcc1cea10744a139c9fddc858a41727, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/126f85d0bb59428da041e6f019a783ab, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/1e0f8f84bb7f41ecbefc0b186c3bd07b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d54a2ec3cf6441aabac7b9efc6024c6a] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=48.1 K 2024-12-06T06:16:41,690 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/77ffdca074784d9eb8427cafda7f30f2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cbcb394ac32c45d28bfdbe016827a496, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/18fa7cf2c9cd4d6494f579a98250a2e4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/8bcb3c40415344eaa1e004a10aeccafe] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=130.6 K 2024-12-06T06:16:41,690 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,690 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. files: [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/77ffdca074784d9eb8427cafda7f30f2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cbcb394ac32c45d28bfdbe016827a496, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/18fa7cf2c9cd4d6494f579a98250a2e4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/8bcb3c40415344eaa1e004a10aeccafe] 2024-12-06T06:16:41,690 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dcc1cea10744a139c9fddc858a41727, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465795555 2024-12-06T06:16:41,691 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ffdca074784d9eb8427cafda7f30f2, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465795555 2024-12-06T06:16:41,691 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 126f85d0bb59428da041e6f019a783ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733465796045 2024-12-06T06:16:41,691 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbcb394ac32c45d28bfdbe016827a496, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733465796045 2024-12-06T06:16:41,691 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0f8f84bb7f41ecbefc0b186c3bd07b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733465796705 2024-12-06T06:16:41,691 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18fa7cf2c9cd4d6494f579a98250a2e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733465796705 2024-12-06T06:16:41,691 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting d54a2ec3cf6441aabac7b9efc6024c6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465798886 2024-12-06T06:16:41,691 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bcb3c40415344eaa1e004a10aeccafe, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465798886 2024-12-06T06:16:41,696 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:41,697 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#B#compaction#542 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:41,697 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206bf86549ec4c943589895f0298535d626_b41c24c7a57a1a19a0dc6ea2edf56813 store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:41,697 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/8967ca1ebc334a4196014ef7a8a3363e is 50, key is test_row_0/B:col10/1733465800037/Put/seqid=0 2024-12-06T06:16:41,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742465_1641 (size=12899) 2024-12-06T06:16:41,702 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206bf86549ec4c943589895f0298535d626_b41c24c7a57a1a19a0dc6ea2edf56813, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:41,702 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206bf86549ec4c943589895f0298535d626_b41c24c7a57a1a19a0dc6ea2edf56813 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:41,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742466_1642 (size=4469) 2024-12-06T06:16:41,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-06T06:16:41,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:41,741 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-06T06:16:41,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:41,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:41,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:41,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:41,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:41,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:41,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063db3bdaba0ab4efcb875c6d19201d4dc_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465800042/Put/seqid=0 2024-12-06T06:16:41,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742467_1643 (size=12454) 2024-12-06T06:16:42,105 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/8967ca1ebc334a4196014ef7a8a3363e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/8967ca1ebc334a4196014ef7a8a3363e 2024-12-06T06:16:42,108 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/B of b41c24c7a57a1a19a0dc6ea2edf56813 into 8967ca1ebc334a4196014ef7a8a3363e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:42,108 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:42,108 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/B, priority=12, startTime=1733465801689; duration=0sec 2024-12-06T06:16:42,108 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T06:16:42,108 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:B 2024-12-06T06:16:42,108 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T06:16:42,109 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49216 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T06:16:42,109 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1541): b41c24c7a57a1a19a0dc6ea2edf56813/C is initiating minor compaction (all files) 2024-12-06T06:16:42,109 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b41c24c7a57a1a19a0dc6ea2edf56813/C in TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:42,109 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6ce6fbcff2404c1ea47fdcaf8339f81f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/971d2ac7728f4f9ab1ee44ad794f26f3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8146ca4f18ad4f6b9a1658268905ee7a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/147e62ff9b564982b53155d1139c4a7b] into tmpdir=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp, totalSize=48.1 K 2024-12-06T06:16:42,110 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce6fbcff2404c1ea47fdcaf8339f81f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733465795555 2024-12-06T06:16:42,110 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 971d2ac7728f4f9ab1ee44ad794f26f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733465796045 2024-12-06T06:16:42,110 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8146ca4f18ad4f6b9a1658268905ee7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733465796705 2024-12-06T06:16:42,110 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#A#compaction#543 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:42,110 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] compactions.Compactor(225): Compacting 147e62ff9b564982b53155d1139c4a7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733465798886 2024-12-06T06:16:42,111 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/f8584beb6a604ac98b1e41e4fa77cb37 is 175, key is test_row_0/A:col10/1733465800037/Put/seqid=0 2024-12-06T06:16:42,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742468_1644 (size=31853) 2024-12-06T06:16:42,117 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b41c24c7a57a1a19a0dc6ea2edf56813#C#compaction#545 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T06:16:42,117 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/50b880b246e34e2aa08967695d3cd397 is 50, key is test_row_0/C:col10/1733465800037/Put/seqid=0 2024-12-06T06:16:42,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742469_1645 (size=12899) 2024-12-06T06:16:42,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:42,153 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063db3bdaba0ab4efcb875c6d19201d4dc_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063db3bdaba0ab4efcb875c6d19201d4dc_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:42,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/b82267cde8a94f4aaaa56dc55783c13e, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:42,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/b82267cde8a94f4aaaa56dc55783c13e is 175, key is test_row_0/A:col10/1733465800042/Put/seqid=0 2024-12-06T06:16:42,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742470_1646 (size=31255) 2024-12-06T06:16:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39987 {}] regionserver.HRegion(8855): Flush requested on b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:42,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. as already flushing 2024-12-06T06:16:42,205 INFO [Thread-2509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2509. 2024-12-06T06:16:42,205 INFO [Thread-2515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2515. 2024-12-06T06:16:42,205 DEBUG [Thread-2509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:42,205 DEBUG [Thread-2515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:42,205 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:42,205 DEBUG [Thread-2515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:42,205 DEBUG [Thread-2515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:42,205 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:42,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:42,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:42,215 INFO [Thread-2511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2511. 2024-12-06T06:16:42,215 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-12-06T06:16:42,215 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:42,215 DEBUG [Thread-2511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:42,215 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:42,215 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:42,215 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:42,215 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:42,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:42,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T06:16:42,517 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/f8584beb6a604ac98b1e41e4fa77cb37 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/f8584beb6a604ac98b1e41e4fa77cb37 2024-12-06T06:16:42,520 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/A of b41c24c7a57a1a19a0dc6ea2edf56813 into f8584beb6a604ac98b1e41e4fa77cb37(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:42,520 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:42,520 INFO [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/A, priority=12, startTime=1733465801689; duration=0sec 2024-12-06T06:16:42,521 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:42,521 DEBUG [RS:0;6336e8474e1c:39987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:A 2024-12-06T06:16:42,523 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/50b880b246e34e2aa08967695d3cd397 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/50b880b246e34e2aa08967695d3cd397 2024-12-06T06:16:42,526 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b41c24c7a57a1a19a0dc6ea2edf56813/C of b41c24c7a57a1a19a0dc6ea2edf56813 into 50b880b246e34e2aa08967695d3cd397(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T06:16:42,526 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:42,526 INFO [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., storeName=b41c24c7a57a1a19a0dc6ea2edf56813/C, priority=12, startTime=1733465801689; duration=0sec 2024-12-06T06:16:42,526 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T06:16:42,526 DEBUG [RS:0;6336e8474e1c:39987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b41c24c7a57a1a19a0dc6ea2edf56813:C 2024-12-06T06:16:42,557 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/b82267cde8a94f4aaaa56dc55783c13e 2024-12-06T06:16:42,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/285066bd10134811a39c363bfe056069 is 50, key is test_row_0/B:col10/1733465800042/Put/seqid=0 2024-12-06T06:16:42,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742471_1647 (size=12301) 2024-12-06T06:16:42,965 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/285066bd10134811a39c363bfe056069 2024-12-06T06:16:42,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/34154aed358c4d8ea07ddda88859b639 is 50, key is test_row_0/C:col10/1733465800042/Put/seqid=0 2024-12-06T06:16:42,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742472_1648 (size=12301) 2024-12-06T06:16:43,373 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/34154aed358c4d8ea07ddda88859b639 2024-12-06T06:16:43,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/b82267cde8a94f4aaaa56dc55783c13e as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/b82267cde8a94f4aaaa56dc55783c13e 2024-12-06T06:16:43,379 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/b82267cde8a94f4aaaa56dc55783c13e, entries=150, sequenceid=274, filesize=30.5 K 2024-12-06T06:16:43,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/285066bd10134811a39c363bfe056069 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/285066bd10134811a39c363bfe056069 2024-12-06T06:16:43,382 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/285066bd10134811a39c363bfe056069, entries=150, sequenceid=274, filesize=12.0 K 2024-12-06T06:16:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/34154aed358c4d8ea07ddda88859b639 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/34154aed358c4d8ea07ddda88859b639 2024-12-06T06:16:43,385 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/34154aed358c4d8ea07ddda88859b639, entries=150, sequenceid=274, filesize=12.0 K 2024-12-06T06:16:43,385 INFO [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1644ms, sequenceid=274, compaction requested=false 2024-12-06T06:16:43,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for b41c24c7a57a1a19a0dc6ea2edf56813: 2024-12-06T06:16:43,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:43,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6336e8474e1c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-06T06:16:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-06T06:16:43,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-06T06:16:43,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0310 sec 2024-12-06T06:16:43,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.0340 sec 2024-12-06T06:16:43,795 INFO [Thread-2513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2513. 2024-12-06T06:16:43,795 DEBUG [Thread-2513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T06:16:43,795 DEBUG [Thread-2513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:43,795 DEBUG [Thread-2513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:43,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T06:16:44,524 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2928 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3050 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2969 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2932 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3050 2024-12-06T06:16:44,524 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T06:16:44,524 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T06:16:44,524 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:16:44,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:44,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:44,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:44,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T06:16:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-06T06:16:44,528 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465804527"}]},"ts":"1733465804527"} 2024-12-06T06:16:44,529 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T06:16:44,529 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T06:16:44,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T06:16:44,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, UNASSIGN}] 2024-12-06T06:16:44,531 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, UNASSIGN 2024-12-06T06:16:44,531 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=b41c24c7a57a1a19a0dc6ea2edf56813, regionState=CLOSING, regionLocation=6336e8474e1c,39987,1733465620995 2024-12-06T06:16:44,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, UNASSIGN because future has completed 2024-12-06T06:16:44,533 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T06:16:44,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995}] 2024-12-06T06:16:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-06T06:16:44,685 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing b41c24c7a57a1a19a0dc6ea2edf56813, disabling compactions & flushes 2024-12-06T06:16:44,685 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. after waiting 0 ms 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:44,685 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing b41c24c7a57a1a19a0dc6ea2edf56813 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=A 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=B 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b41c24c7a57a1a19a0dc6ea2edf56813, store=C 2024-12-06T06:16:44,685 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-06T06:16:44,690 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206aaa8ed43336e4fee9c1ca9189cb64035_b41c24c7a57a1a19a0dc6ea2edf56813 is 50, key is test_row_0/A:col10/1733465802214/Put/seqid=0 2024-12-06T06:16:44,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742473_1649 (size=9914) 2024-12-06T06:16:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-06T06:16:45,093 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T06:16:45,096 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206aaa8ed43336e4fee9c1ca9189cb64035_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206aaa8ed43336e4fee9c1ca9189cb64035_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:45,096 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/01c90593ce474ee7a430fcc4bb38f817, store: [table=TestAcidGuarantees family=A region=b41c24c7a57a1a19a0dc6ea2edf56813] 2024-12-06T06:16:45,097 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/01c90593ce474ee7a430fcc4bb38f817 is 175, key is test_row_0/A:col10/1733465802214/Put/seqid=0 2024-12-06T06:16:45,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742474_1650 (size=22561) 2024-12-06T06:16:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-06T06:16:45,500 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/01c90593ce474ee7a430fcc4bb38f817 2024-12-06T06:16:45,505 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/649f87ab58094c489721d7c42095265c is 50, key is test_row_0/B:col10/1733465802214/Put/seqid=0 2024-12-06T06:16:45,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742475_1651 (size=9857) 2024-12-06T06:16:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-06T06:16:45,908 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/649f87ab58094c489721d7c42095265c 2024-12-06T06:16:45,913 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/10930ca9beaa4ccd9e9ec25c329600c9 is 50, key is test_row_0/C:col10/1733465802214/Put/seqid=0 2024-12-06T06:16:45,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742476_1652 (size=9857) 2024-12-06T06:16:46,316 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/10930ca9beaa4ccd9e9ec25c329600c9 2024-12-06T06:16:46,319 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/A/01c90593ce474ee7a430fcc4bb38f817 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/01c90593ce474ee7a430fcc4bb38f817 2024-12-06T06:16:46,322 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/01c90593ce474ee7a430fcc4bb38f817, entries=100, sequenceid=285, filesize=22.0 K 2024-12-06T06:16:46,322 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/B/649f87ab58094c489721d7c42095265c as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/649f87ab58094c489721d7c42095265c 2024-12-06T06:16:46,325 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/649f87ab58094c489721d7c42095265c, entries=100, sequenceid=285, filesize=9.6 K 2024-12-06T06:16:46,325 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/.tmp/C/10930ca9beaa4ccd9e9ec25c329600c9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/10930ca9beaa4ccd9e9ec25c329600c9 2024-12-06T06:16:46,328 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/10930ca9beaa4ccd9e9ec25c329600c9, entries=100, sequenceid=285, filesize=9.6 K 2024-12-06T06:16:46,328 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1643ms, sequenceid=285, compaction requested=true 2024-12-06T06:16:46,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/16f0bf2143744213ba3e22f535fc01d1, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/11c5fa5f9d3246e9968ca51155a0c382, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/729cfb3dbfb8411ebd9120371ff3ccb6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1455a9e11fc42e9808c4b1fa7fc2028, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/d5ac6ae428e84941b28767c3a34f8374, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7200d2c0ed0e4e8dbb21e8e54b5f53ea, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/5fd54b9c9f054604a529b6eb4f54434b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7346ec26460d45f3871fb62da1e7b5cb, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/fb2859b7baa04b598b477a020e6fe537, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3c495fbf5c51452e967fae01b269059b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1f1e970a0c543c9a8552f24fb17ef96, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/54ab6649cb004394a40ae59680d7125b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3a5ec64cc5e446a681ecc93cc5544af3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/1a196e4d18784815ab458147a670ab74, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cb7c94107661487c92c7f06e37c67d6d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/77ffdca074784d9eb8427cafda7f30f2, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cbcb394ac32c45d28bfdbe016827a496, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/18fa7cf2c9cd4d6494f579a98250a2e4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/8bcb3c40415344eaa1e004a10aeccafe] to archive 2024-12-06T06:16:46,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:16:46,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/16f0bf2143744213ba3e22f535fc01d1 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/16f0bf2143744213ba3e22f535fc01d1 2024-12-06T06:16:46,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/11c5fa5f9d3246e9968ca51155a0c382 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/11c5fa5f9d3246e9968ca51155a0c382 2024-12-06T06:16:46,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/729cfb3dbfb8411ebd9120371ff3ccb6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/729cfb3dbfb8411ebd9120371ff3ccb6 2024-12-06T06:16:46,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1455a9e11fc42e9808c4b1fa7fc2028 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1455a9e11fc42e9808c4b1fa7fc2028 2024-12-06T06:16:46,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/d5ac6ae428e84941b28767c3a34f8374 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/d5ac6ae428e84941b28767c3a34f8374 2024-12-06T06:16:46,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7200d2c0ed0e4e8dbb21e8e54b5f53ea to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7200d2c0ed0e4e8dbb21e8e54b5f53ea 2024-12-06T06:16:46,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/5fd54b9c9f054604a529b6eb4f54434b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/5fd54b9c9f054604a529b6eb4f54434b 2024-12-06T06:16:46,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7346ec26460d45f3871fb62da1e7b5cb to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/7346ec26460d45f3871fb62da1e7b5cb 2024-12-06T06:16:46,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/fb2859b7baa04b598b477a020e6fe537 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/fb2859b7baa04b598b477a020e6fe537 2024-12-06T06:16:46,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3c495fbf5c51452e967fae01b269059b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3c495fbf5c51452e967fae01b269059b 2024-12-06T06:16:46,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1f1e970a0c543c9a8552f24fb17ef96 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/e1f1e970a0c543c9a8552f24fb17ef96 2024-12-06T06:16:46,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/54ab6649cb004394a40ae59680d7125b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/54ab6649cb004394a40ae59680d7125b 2024-12-06T06:16:46,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3a5ec64cc5e446a681ecc93cc5544af3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/3a5ec64cc5e446a681ecc93cc5544af3 2024-12-06T06:16:46,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/1a196e4d18784815ab458147a670ab74 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/1a196e4d18784815ab458147a670ab74 2024-12-06T06:16:46,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cb7c94107661487c92c7f06e37c67d6d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cb7c94107661487c92c7f06e37c67d6d 2024-12-06T06:16:46,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/77ffdca074784d9eb8427cafda7f30f2 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/77ffdca074784d9eb8427cafda7f30f2 2024-12-06T06:16:46,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cbcb394ac32c45d28bfdbe016827a496 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/cbcb394ac32c45d28bfdbe016827a496 2024-12-06T06:16:46,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/18fa7cf2c9cd4d6494f579a98250a2e4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/18fa7cf2c9cd4d6494f579a98250a2e4 2024-12-06T06:16:46,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/8bcb3c40415344eaa1e004a10aeccafe to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/8bcb3c40415344eaa1e004a10aeccafe 2024-12-06T06:16:46,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/96216d83ebc24ed6bee431d20b0a2d43, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/f4716d3dcdc8410bba3776d022b00e3a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5ab66167b95e4ef39968598bfa6e5db6, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/89c9b430e75a4aed84ba3d3c87c97154, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d10342bcb6ca451ab81c919e602be704, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/24032255b0ad470384c7761c773deb7e, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5c7d35356e314e1a95545b391b9e1afc, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/54aefb14d9e44a4084dff1530088c246, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/e6e7749b61bf456c8b53f13cce1f2eee, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/6d3c52b98f274dc38ca35b9e2520f0e4, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/97f53005c2e74ee8b572303bb324473c, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/a1571d7694634fbcb1fc03cbae26f03b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/77ada57b72324ab8b0c6fa3cbd7ec4a7, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/563d96e490d84878b829af364bf04421, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/3dcc1cea10744a139c9fddc858a41727, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/45f7258869f14c60afeb16ed28ea6612, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/126f85d0bb59428da041e6f019a783ab, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/1e0f8f84bb7f41ecbefc0b186c3bd07b, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d54a2ec3cf6441aabac7b9efc6024c6a] to archive 2024-12-06T06:16:46,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:16:46,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/96216d83ebc24ed6bee431d20b0a2d43 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/96216d83ebc24ed6bee431d20b0a2d43 2024-12-06T06:16:46,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/f4716d3dcdc8410bba3776d022b00e3a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/f4716d3dcdc8410bba3776d022b00e3a 2024-12-06T06:16:46,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5ab66167b95e4ef39968598bfa6e5db6 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5ab66167b95e4ef39968598bfa6e5db6 2024-12-06T06:16:46,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/89c9b430e75a4aed84ba3d3c87c97154 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/89c9b430e75a4aed84ba3d3c87c97154 2024-12-06T06:16:46,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d10342bcb6ca451ab81c919e602be704 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d10342bcb6ca451ab81c919e602be704 2024-12-06T06:16:46,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/24032255b0ad470384c7761c773deb7e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/24032255b0ad470384c7761c773deb7e 2024-12-06T06:16:46,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5c7d35356e314e1a95545b391b9e1afc to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/5c7d35356e314e1a95545b391b9e1afc 2024-12-06T06:16:46,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/54aefb14d9e44a4084dff1530088c246 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/54aefb14d9e44a4084dff1530088c246 2024-12-06T06:16:46,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/e6e7749b61bf456c8b53f13cce1f2eee to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/e6e7749b61bf456c8b53f13cce1f2eee 2024-12-06T06:16:46,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/6d3c52b98f274dc38ca35b9e2520f0e4 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/6d3c52b98f274dc38ca35b9e2520f0e4 2024-12-06T06:16:46,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/97f53005c2e74ee8b572303bb324473c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/97f53005c2e74ee8b572303bb324473c 2024-12-06T06:16:46,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/a1571d7694634fbcb1fc03cbae26f03b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/a1571d7694634fbcb1fc03cbae26f03b 2024-12-06T06:16:46,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/77ada57b72324ab8b0c6fa3cbd7ec4a7 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/77ada57b72324ab8b0c6fa3cbd7ec4a7 2024-12-06T06:16:46,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/563d96e490d84878b829af364bf04421 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/563d96e490d84878b829af364bf04421 2024-12-06T06:16:46,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/3dcc1cea10744a139c9fddc858a41727 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/3dcc1cea10744a139c9fddc858a41727 2024-12-06T06:16:46,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/45f7258869f14c60afeb16ed28ea6612 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/45f7258869f14c60afeb16ed28ea6612 2024-12-06T06:16:46,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/126f85d0bb59428da041e6f019a783ab to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/126f85d0bb59428da041e6f019a783ab 2024-12-06T06:16:46,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/1e0f8f84bb7f41ecbefc0b186c3bd07b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/1e0f8f84bb7f41ecbefc0b186c3bd07b 2024-12-06T06:16:46,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d54a2ec3cf6441aabac7b9efc6024c6a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/d54a2ec3cf6441aabac7b9efc6024c6a 2024-12-06T06:16:46,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/68ac8acce4024e2eb6df2aeb3745a503, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0d51a5c859b149a093bb9ecdaedcc6ee, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0f511a4927954126840e810acc72b64d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6a49b06ca95c422994c0835603b44250, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/813f460b6a8044b2b2c09785d3704985, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/9294085e68d54256800d82e79ade5f72, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0ebc092d1d4402889c1a498b9dc5247, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/082a5298fb034a0fbe9cf0c1d246b511, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8addfbcb23554a70a686b791e15b2477, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0729fb3118d4942b6396ea0c52c5632, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/c3c123308c244139812e2951cd2dd662, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/59b7862f14194142a5beb7aa9f69158d, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/f2716104045149c997ef39fbf840bbda, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/5f0eb8edd20f41fe924dcc618de6e145, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6ce6fbcff2404c1ea47fdcaf8339f81f, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/3c8ab526755c4ae6a83aa127cbc81736, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/971d2ac7728f4f9ab1ee44ad794f26f3, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8146ca4f18ad4f6b9a1658268905ee7a, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/147e62ff9b564982b53155d1139c4a7b] to archive 2024-12-06T06:16:46,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T06:16:46,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/68ac8acce4024e2eb6df2aeb3745a503 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/68ac8acce4024e2eb6df2aeb3745a503 2024-12-06T06:16:46,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0d51a5c859b149a093bb9ecdaedcc6ee to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0d51a5c859b149a093bb9ecdaedcc6ee 2024-12-06T06:16:46,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0f511a4927954126840e810acc72b64d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/0f511a4927954126840e810acc72b64d 2024-12-06T06:16:46,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6a49b06ca95c422994c0835603b44250 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6a49b06ca95c422994c0835603b44250 2024-12-06T06:16:46,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/813f460b6a8044b2b2c09785d3704985 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/813f460b6a8044b2b2c09785d3704985 2024-12-06T06:16:46,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/9294085e68d54256800d82e79ade5f72 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/9294085e68d54256800d82e79ade5f72 2024-12-06T06:16:46,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0ebc092d1d4402889c1a498b9dc5247 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0ebc092d1d4402889c1a498b9dc5247 2024-12-06T06:16:46,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/082a5298fb034a0fbe9cf0c1d246b511 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/082a5298fb034a0fbe9cf0c1d246b511 2024-12-06T06:16:46,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8addfbcb23554a70a686b791e15b2477 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8addfbcb23554a70a686b791e15b2477 2024-12-06T06:16:46,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0729fb3118d4942b6396ea0c52c5632 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/b0729fb3118d4942b6396ea0c52c5632 2024-12-06T06:16:46,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/c3c123308c244139812e2951cd2dd662 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/c3c123308c244139812e2951cd2dd662 2024-12-06T06:16:46,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/59b7862f14194142a5beb7aa9f69158d to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/59b7862f14194142a5beb7aa9f69158d 2024-12-06T06:16:46,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/f2716104045149c997ef39fbf840bbda to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/f2716104045149c997ef39fbf840bbda 2024-12-06T06:16:46,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/5f0eb8edd20f41fe924dcc618de6e145 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/5f0eb8edd20f41fe924dcc618de6e145 2024-12-06T06:16:46,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6ce6fbcff2404c1ea47fdcaf8339f81f to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/6ce6fbcff2404c1ea47fdcaf8339f81f 2024-12-06T06:16:46,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/3c8ab526755c4ae6a83aa127cbc81736 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/3c8ab526755c4ae6a83aa127cbc81736 2024-12-06T06:16:46,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/971d2ac7728f4f9ab1ee44ad794f26f3 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/971d2ac7728f4f9ab1ee44ad794f26f3 2024-12-06T06:16:46,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8146ca4f18ad4f6b9a1658268905ee7a to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/8146ca4f18ad4f6b9a1658268905ee7a 2024-12-06T06:16:46,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/147e62ff9b564982b53155d1139c4a7b to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/147e62ff9b564982b53155d1139c4a7b 2024-12-06T06:16:46,378 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/recovered.edits/288.seqid, newMaxSeqId=288, maxSeqId=4 2024-12-06T06:16:46,378 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. 2024-12-06T06:16:46,378 DEBUG [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for b41c24c7a57a1a19a0dc6ea2edf56813: Waiting for close lock at 1733465804685Running coprocessor pre-close hooks at 1733465804685Disabling compacts and flushes for region at 1733465804685Disabling writes for close at 1733465804685Obtaining lock to block concurrent updates at 1733465804685Preparing flush snapshotting stores in b41c24c7a57a1a19a0dc6ea2edf56813 at 1733465804685Finished memstore snapshotting TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733465804685Flushing stores of TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813. at 1733465804686 (+1 ms)Flushing b41c24c7a57a1a19a0dc6ea2edf56813/A: creating writer at 1733465804686Flushing mob file b41c24c7a57a1a19a0dc6ea2edf56813/A: appending metadata at 1733465804689 (+3 ms)Flushing mob file b41c24c7a57a1a19a0dc6ea2edf56813/A: closing flushed file at 1733465804689Flushing b41c24c7a57a1a19a0dc6ea2edf56813/A: appending metadata at 1733465805096 (+407 ms)Flushing b41c24c7a57a1a19a0dc6ea2edf56813/A: closing flushed file at 1733465805096Flushing b41c24c7a57a1a19a0dc6ea2edf56813/B: creating writer at 1733465805503 (+407 ms)Flushing b41c24c7a57a1a19a0dc6ea2edf56813/B: appending metadata at 1733465805504 (+1 ms)Flushing b41c24c7a57a1a19a0dc6ea2edf56813/B: closing flushed file at 1733465805504Flushing b41c24c7a57a1a19a0dc6ea2edf56813/C: creating writer at 1733465805911 (+407 ms)Flushing b41c24c7a57a1a19a0dc6ea2edf56813/C: appending metadata at 1733465805912 (+1 ms)Flushing b41c24c7a57a1a19a0dc6ea2edf56813/C: closing flushed file at 1733465805913 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66faad60: reopening flushed file at 1733465806319 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7756e1fe: reopening flushed file at 1733465806322 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@272e40ce: reopening flushed file at 1733465806325 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b41c24c7a57a1a19a0dc6ea2edf56813 in 1643ms, sequenceid=285, compaction requested=true at 1733465806328 (+3 ms)Writing region close event to WAL at 1733465806376 (+48 ms)Running coprocessor post-close hooks at 1733465806378 (+2 ms)Closed at 1733465806378 2024-12-06T06:16:46,380 INFO [RS_CLOSE_REGION-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,381 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=b41c24c7a57a1a19a0dc6ea2edf56813, regionState=CLOSED 2024-12-06T06:16:46,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 because future has completed 2024-12-06T06:16:46,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-06T06:16:46,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure b41c24c7a57a1a19a0dc6ea2edf56813, server=6336e8474e1c,39987,1733465620995 in 1.8490 sec 2024-12-06T06:16:46,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-12-06T06:16:46,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b41c24c7a57a1a19a0dc6ea2edf56813, UNASSIGN in 1.8530 sec 2024-12-06T06:16:46,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-06T06:16:46,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8560 sec 2024-12-06T06:16:46,387 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733465806387"}]},"ts":"1733465806387"} 2024-12-06T06:16:46,388 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T06:16:46,388 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T06:16:46,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8630 sec 2024-12-06T06:16:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-06T06:16:46,674 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:46,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T06:16:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:46,676 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-06T06:16:46,677 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:46,678 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,680 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C, FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/recovered.edits] 2024-12-06T06:16:46,682 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/01c90593ce474ee7a430fcc4bb38f817 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/01c90593ce474ee7a430fcc4bb38f817 2024-12-06T06:16:46,682 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/b82267cde8a94f4aaaa56dc55783c13e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/b82267cde8a94f4aaaa56dc55783c13e 2024-12-06T06:16:46,684 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/f8584beb6a604ac98b1e41e4fa77cb37 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/A/f8584beb6a604ac98b1e41e4fa77cb37 2024-12-06T06:16:46,685 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/285066bd10134811a39c363bfe056069 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/285066bd10134811a39c363bfe056069 2024-12-06T06:16:46,686 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/649f87ab58094c489721d7c42095265c to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/649f87ab58094c489721d7c42095265c 2024-12-06T06:16:46,687 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/8967ca1ebc334a4196014ef7a8a3363e to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/B/8967ca1ebc334a4196014ef7a8a3363e 2024-12-06T06:16:46,688 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/10930ca9beaa4ccd9e9ec25c329600c9 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/10930ca9beaa4ccd9e9ec25c329600c9 2024-12-06T06:16:46,689 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/34154aed358c4d8ea07ddda88859b639 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/34154aed358c4d8ea07ddda88859b639 2024-12-06T06:16:46,690 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/50b880b246e34e2aa08967695d3cd397 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/C/50b880b246e34e2aa08967695d3cd397 2024-12-06T06:16:46,691 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/recovered.edits/288.seqid to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813/recovered.edits/288.seqid 2024-12-06T06:16:46,692 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/default/TestAcidGuarantees/b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,692 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T06:16:46,692 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T06:16:46,692 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T06:16:46,694 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206165997eca1b945eea306cacbd7f21c88_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206165997eca1b945eea306cacbd7f21c88_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,695 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120623fb80aeb31b4b1592d33965b7c6480b_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120623fb80aeb31b4b1592d33965b7c6480b_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,696 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206248c45af866d46b28144eec123f72d84_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206248c45af866d46b28144eec123f72d84_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,696 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206259c362f782d4706b645461021c747ac_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206259c362f782d4706b645461021c747ac_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,697 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120626c66874fdc64c2db8f7afa4b6988951_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120626c66874fdc64c2db8f7afa4b6988951_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,698 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b1e0beb15d24fc28541e9486fd8158b_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b1e0beb15d24fc28541e9486fd8158b_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,699 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b3325ac8a4248aa80130098af384b67_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062b3325ac8a4248aa80130098af384b67_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,699 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063db3bdaba0ab4efcb875c6d19201d4dc_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063db3bdaba0ab4efcb875c6d19201d4dc_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,700 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066d809d4b01d04aecadd714800bcffa54_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066d809d4b01d04aecadd714800bcffa54_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,701 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206aaa8ed43336e4fee9c1ca9189cb64035_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206aaa8ed43336e4fee9c1ca9189cb64035_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,702 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ab0792204d134b5889b1661cecce156a_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ab0792204d134b5889b1661cecce156a_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,702 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ced23571996f49caa935dc09d4a9769d_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ced23571996f49caa935dc09d4a9769d_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,703 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ed12c0be46ce480a9e63043c529af290_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ed12c0be46ce480a9e63043c529af290_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,704 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eeb540db78924281a7dbd92141ab0787_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206eeb540db78924281a7dbd92141ab0787_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,704 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f6e816a5c5014fb98faf260cfdc4e5af_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f6e816a5c5014fb98faf260cfdc4e5af_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,705 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ff7347b8c6384a03a90735a300bd150a_b41c24c7a57a1a19a0dc6ea2edf56813 to hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ff7347b8c6384a03a90735a300bd150a_b41c24c7a57a1a19a0dc6ea2edf56813 2024-12-06T06:16:46,706 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T06:16:46,707 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:46,709 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T06:16:46,711 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T06:16:46,711 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:46,711 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T06:16:46,712 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733465806711"}]},"ts":"9223372036854775807"} 2024-12-06T06:16:46,713 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T06:16:46,713 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b41c24c7a57a1a19a0dc6ea2edf56813, NAME => 'TestAcidGuarantees,,1733465778282.b41c24c7a57a1a19a0dc6ea2edf56813.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T06:16:46,713 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T06:16:46,713 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733465806713"}]},"ts":"9223372036854775807"} 2024-12-06T06:16:46,714 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T06:16:46,715 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T06:16:46,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-12-06T06:16:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-06T06:16:46,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T06:16:46,784 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T06:16:46,794 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249 (was 249), OpenFileDescriptor=447 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=333 (was 347), ProcessCount=11 (was 11), AvailableMemoryMB=4166 (was 4235) 2024-12-06T06:16:46,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T06:16:46,794 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T06:16:46,794 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:16:46,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:46,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:46,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T06:16:46,795 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T06:16:46,795 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1143773905, stopped=false 2024-12-06T06:16:46,795 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6336e8474e1c,40959,1733465620259 2024-12-06T06:16:46,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T06:16:46,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T06:16:46,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:16:46,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:16:46,797 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T06:16:46,798 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T06:16:46,798 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:16:46,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:46,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T06:16:46,798 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6336e8474e1c,39987,1733465620995' ***** 2024-12-06T06:16:46,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T06:16:46,798 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T06:16:46,799 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T06:16:46,799 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T06:16:46,799 INFO [RS:0;6336e8474e1c:39987 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T06:16:46,799 INFO [RS:0;6336e8474e1c:39987 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T06:16:46,799 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(959): stopping server 6336e8474e1c,39987,1733465620995 2024-12-06T06:16:46,799 INFO [RS:0;6336e8474e1c:39987 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T06:16:46,799 INFO [RS:0;6336e8474e1c:39987 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6336e8474e1c:39987. 2024-12-06T06:16:46,799 DEBUG [RS:0;6336e8474e1c:39987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T06:16:46,799 DEBUG [RS:0;6336e8474e1c:39987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:46,800 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T06:16:46,800 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T06:16:46,800 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T06:16:46,800 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T06:16:46,800 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-06T06:16:46,801 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-06T06:16:46,801 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T06:16:46,801 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:46,801 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T06:16:46,801 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T06:16:46,801 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T06:16:46,801 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T06:16:46,801 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-06T06:16:46,825 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/info/7b93ba55bae2478f8427ba1578474b52 is 95, key is TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d./info:/1733465653796/DeleteFamily/seqid=0 2024-12-06T06:16:46,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742477_1653 (size=6514) 2024-12-06T06:16:46,858 INFO [regionserver/6336e8474e1c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T06:16:47,001 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:47,201 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:47,229 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/info/7b93ba55bae2478f8427ba1578474b52 2024-12-06T06:16:47,251 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/ns/e08543491e6c488e9aefd9acca86054d is 93, key is TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d./ns:/1733465653756/DeleteFamily/seqid=0 2024-12-06T06:16:47,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742478_1654 (size=5906) 2024-12-06T06:16:47,401 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:47,536 INFO [regionserver/6336e8474e1c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T06:16:47,536 INFO [regionserver/6336e8474e1c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T06:16:47,602 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:47,655 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/ns/e08543491e6c488e9aefd9acca86054d 2024-12-06T06:16:47,672 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/rep_barrier/791820ab4cbd4cb9b7b9caf5063178c9 is 102, key is TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d./rep_barrier:/1733465653756/DeleteFamily/seqid=0 2024-12-06T06:16:47,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742479_1655 (size=6025) 2024-12-06T06:16:47,802 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:48,002 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:48,075 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/rep_barrier/791820ab4cbd4cb9b7b9caf5063178c9 2024-12-06T06:16:48,092 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/table/88bb5d74fe40426a9333404fdffa7efa is 96, key is TestAcidGuarantees,,1733465625156.3860e7c2497c4799e160cda168e0086d./table:/1733465653756/DeleteFamily/seqid=0 2024-12-06T06:16:48,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742480_1656 (size=5936) 2024-12-06T06:16:48,202 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:48,402 DEBUG [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T06:16:48,496 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/table/88bb5d74fe40426a9333404fdffa7efa 2024-12-06T06:16:48,499 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/info/7b93ba55bae2478f8427ba1578474b52 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/info/7b93ba55bae2478f8427ba1578474b52 2024-12-06T06:16:48,502 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/info/7b93ba55bae2478f8427ba1578474b52, entries=12, sequenceid=90, filesize=6.4 K 2024-12-06T06:16:48,503 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/ns/e08543491e6c488e9aefd9acca86054d as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/ns/e08543491e6c488e9aefd9acca86054d 2024-12-06T06:16:48,505 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/ns/e08543491e6c488e9aefd9acca86054d, entries=8, sequenceid=90, filesize=5.8 K 2024-12-06T06:16:48,506 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/rep_barrier/791820ab4cbd4cb9b7b9caf5063178c9 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/rep_barrier/791820ab4cbd4cb9b7b9caf5063178c9 2024-12-06T06:16:48,508 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/rep_barrier/791820ab4cbd4cb9b7b9caf5063178c9, entries=6, sequenceid=90, filesize=5.9 K 2024-12-06T06:16:48,509 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/.tmp/table/88bb5d74fe40426a9333404fdffa7efa as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/table/88bb5d74fe40426a9333404fdffa7efa 2024-12-06T06:16:48,511 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/table/88bb5d74fe40426a9333404fdffa7efa, entries=7, sequenceid=90, filesize=5.8 K 2024-12-06T06:16:48,512 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1711ms, sequenceid=90, compaction requested=false 2024-12-06T06:16:48,515 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-06T06:16:48,516 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T06:16:48,516 INFO [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T06:16:48,516 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733465806801Running coprocessor pre-close hooks at 1733465806801Disabling compacts and flushes for region at 1733465806801Disabling writes for close at 1733465806801Obtaining lock to block concurrent updates at 1733465806801Preparing flush snapshotting stores in 1588230740 at 1733465806801Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733465806802 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733465806802Flushing 1588230740/info: creating writer at 1733465806802Flushing 1588230740/info: appending metadata at 1733465806825 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733465806825Flushing 1588230740/ns: creating writer at 1733465807233 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733465807250 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733465807250Flushing 1588230740/rep_barrier: creating writer at 1733465807658 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733465807672 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733465807672Flushing 1588230740/table: creating writer at 1733465808078 (+406 ms)Flushing 1588230740/table: appending metadata at 1733465808092 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733465808092Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f8886ee: reopening flushed file at 1733465808499 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f00bf72: reopening flushed file at 1733465808502 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@598976b2: reopening flushed file at 1733465808505 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cd33602: reopening flushed file at 1733465808508 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1711ms, sequenceid=90, compaction requested=false at 1733465808512 (+4 ms)Writing region close event to WAL at 1733465808513 (+1 ms)Running coprocessor post-close hooks at 1733465808516 (+3 ms)Closed at 1733465808516 2024-12-06T06:16:48,516 DEBUG [RS_CLOSE_META-regionserver/6336e8474e1c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T06:16:48,603 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(976): stopping server 6336e8474e1c,39987,1733465620995; all regions closed. 2024-12-06T06:16:48,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741834_1010 (size=25344) 2024-12-06T06:16:48,609 DEBUG [RS:0;6336e8474e1c:39987 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/oldWALs 2024-12-06T06:16:48,609 INFO [RS:0;6336e8474e1c:39987 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6336e8474e1c%2C39987%2C1733465620995.meta:.meta(num 1733465624669) 2024-12-06T06:16:48,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741832_1008 (size=17531756) 2024-12-06T06:16:48,612 DEBUG [RS:0;6336e8474e1c:39987 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/oldWALs 2024-12-06T06:16:48,612 INFO [RS:0;6336e8474e1c:39987 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6336e8474e1c%2C39987%2C1733465620995:(num 1733465623703) 2024-12-06T06:16:48,612 DEBUG [RS:0;6336e8474e1c:39987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T06:16:48,612 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T06:16:48,612 INFO [RS:0;6336e8474e1c:39987 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T06:16:48,612 INFO [RS:0;6336e8474e1c:39987 {}] hbase.ChoreService(370): Chore service for: regionserver/6336e8474e1c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T06:16:48,613 INFO [RS:0;6336e8474e1c:39987 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T06:16:48,613 INFO [regionserver/6336e8474e1c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T06:16:48,613 INFO [RS:0;6336e8474e1c:39987 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39987 2024-12-06T06:16:48,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6336e8474e1c,39987,1733465620995 2024-12-06T06:16:48,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T06:16:48,617 INFO [RS:0;6336e8474e1c:39987 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T06:16:48,621 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6336e8474e1c,39987,1733465620995] 2024-12-06T06:16:48,622 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6336e8474e1c,39987,1733465620995 already deleted, retry=false 2024-12-06T06:16:48,622 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6336e8474e1c,39987,1733465620995 expired; onlineServers=0 2024-12-06T06:16:48,622 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6336e8474e1c,40959,1733465620259' ***** 2024-12-06T06:16:48,622 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T06:16:48,622 INFO [M:0;6336e8474e1c:40959 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T06:16:48,623 INFO [M:0;6336e8474e1c:40959 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T06:16:48,623 DEBUG [M:0;6336e8474e1c:40959 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T06:16:48,623 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T06:16:48,623 DEBUG [M:0;6336e8474e1c:40959 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T06:16:48,623 DEBUG [master/6336e8474e1c:0:becomeActiveMaster-HFileCleaner.large.0-1733465623339 {}] cleaner.HFileCleaner(306): Exit Thread[master/6336e8474e1c:0:becomeActiveMaster-HFileCleaner.large.0-1733465623339,5,FailOnTimeoutGroup] 2024-12-06T06:16:48,623 DEBUG [master/6336e8474e1c:0:becomeActiveMaster-HFileCleaner.small.0-1733465623340 {}] cleaner.HFileCleaner(306): Exit Thread[master/6336e8474e1c:0:becomeActiveMaster-HFileCleaner.small.0-1733465623340,5,FailOnTimeoutGroup] 2024-12-06T06:16:48,623 INFO [M:0;6336e8474e1c:40959 {}] hbase.ChoreService(370): Chore service for: master/6336e8474e1c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T06:16:48,623 INFO [M:0;6336e8474e1c:40959 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T06:16:48,623 DEBUG [M:0;6336e8474e1c:40959 {}] master.HMaster(1795): Stopping service threads 2024-12-06T06:16:48,623 INFO [M:0;6336e8474e1c:40959 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T06:16:48,623 INFO [M:0;6336e8474e1c:40959 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T06:16:48,623 ERROR [M:0;6336e8474e1c:40959 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-06T06:16:48,624 INFO [M:0;6336e8474e1c:40959 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T06:16:48,624 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T06:16:48,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T06:16:48,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T06:16:48,625 DEBUG [M:0;6336e8474e1c:40959 {}] zookeeper.ZKUtil(347): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T06:16:48,625 WARN [M:0;6336e8474e1c:40959 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T06:16:48,625 INFO [M:0;6336e8474e1c:40959 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/.lastflushedseqids 2024-12-06T06:16:48,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742481_1657 (size=72) 2024-12-06T06:16:48,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T06:16:48,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39987-0x1018668262d0001, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T06:16:48,721 INFO [RS:0;6336e8474e1c:39987 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T06:16:48,722 INFO [RS:0;6336e8474e1c:39987 {}] regionserver.HRegionServer(1031): Exiting; stopping=6336e8474e1c,39987,1733465620995; zookeeper connection closed. 2024-12-06T06:16:48,722 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7cbc272d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7cbc272d 2024-12-06T06:16:48,722 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-06T06:16:49,034 INFO [M:0;6336e8474e1c:40959 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T06:16:49,035 INFO [M:0;6336e8474e1c:40959 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T06:16:49,035 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T06:16:49,035 INFO [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T06:16:49,035 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T06:16:49,035 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T06:16:49,035 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T06:16:49,035 INFO [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=757.88 KB heapSize=927.20 KB 2024-12-06T06:16:49,051 DEBUG [M:0;6336e8474e1c:40959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd19b6b88d214111b8eba0993820f88a is 82, key is hbase:meta,,1/info:regioninfo/1733465624759/Put/seqid=0 2024-12-06T06:16:49,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742482_1658 (size=5672) 2024-12-06T06:16:49,454 INFO [M:0;6336e8474e1c:40959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2087 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd19b6b88d214111b8eba0993820f88a 2024-12-06T06:16:49,474 DEBUG [M:0;6336e8474e1c:40959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f63b9f7b21d4d9cb80ec20e70e696f2 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1733465781345/Put/seqid=0 2024-12-06T06:16:49,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742483_1659 (size=44993) 2024-12-06T06:16:49,878 INFO [M:0;6336e8474e1c:40959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=757.32 KB at sequenceid=2087 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f63b9f7b21d4d9cb80ec20e70e696f2 2024-12-06T06:16:49,882 INFO [M:0;6336e8474e1c:40959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f63b9f7b21d4d9cb80ec20e70e696f2 2024-12-06T06:16:49,902 DEBUG [M:0;6336e8474e1c:40959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6937ad819981416e808bb304e8a725fd is 69, key is 6336e8474e1c,39987,1733465620995/rs:state/1733465623466/Put/seqid=0 2024-12-06T06:16:49,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073742484_1660 (size=5156) 2024-12-06T06:16:50,306 INFO [M:0;6336e8474e1c:40959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2087 (bloomFilter=true), to=hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6937ad819981416e808bb304e8a725fd 2024-12-06T06:16:50,309 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd19b6b88d214111b8eba0993820f88a as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd19b6b88d214111b8eba0993820f88a 2024-12-06T06:16:50,312 INFO [M:0;6336e8474e1c:40959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd19b6b88d214111b8eba0993820f88a, entries=8, sequenceid=2087, filesize=5.5 K 2024-12-06T06:16:50,313 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f63b9f7b21d4d9cb80ec20e70e696f2 as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f63b9f7b21d4d9cb80ec20e70e696f2 2024-12-06T06:16:50,316 INFO [M:0;6336e8474e1c:40959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f63b9f7b21d4d9cb80ec20e70e696f2 2024-12-06T06:16:50,316 INFO [M:0;6336e8474e1c:40959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f63b9f7b21d4d9cb80ec20e70e696f2, entries=168, sequenceid=2087, filesize=43.9 K 2024-12-06T06:16:50,317 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6937ad819981416e808bb304e8a725fd as hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6937ad819981416e808bb304e8a725fd 2024-12-06T06:16:50,319 INFO [M:0;6336e8474e1c:40959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37641/user/jenkins/test-data/b153107e-96a1-ed4b-3871-1ae56a3b5fea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6937ad819981416e808bb304e8a725fd, entries=1, sequenceid=2087, filesize=5.0 K 2024-12-06T06:16:50,320 INFO [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(3140): Finished flush of dataSize ~757.88 KB/776068, heapSize ~926.90 KB/949144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2087, compaction requested=false 2024-12-06T06:16:50,321 INFO [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T06:16:50,321 DEBUG [M:0;6336e8474e1c:40959 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733465809035Disabling compacts and flushes for region at 1733465809035Disabling writes for close at 1733465809035Obtaining lock to block concurrent updates at 1733465809035Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733465809035Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=776068, getHeapSize=949384, getOffHeapSize=0, getCellsCount=2280 at 1733465809035Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733465809036 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733465809036Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733465809050 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733465809050Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733465809458 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733465809474 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733465809474Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733465809882 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733465809902 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733465809902Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a287b19: reopening flushed file at 1733465810309 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44714287: reopening flushed file at 1733465810312 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31c8a16: reopening flushed file at 1733465810316 (+4 ms)Finished flush of dataSize ~757.88 KB/776068, heapSize ~926.90 KB/949144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2087, compaction requested=false at 1733465810320 (+4 ms)Writing region close event to WAL at 1733465810321 (+1 ms)Closed at 1733465810321 2024-12-06T06:16:50,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38749 is added to blk_1073741830_1006 (size=912914) 2024-12-06T06:16:50,324 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T06:16:50,324 INFO [M:0;6336e8474e1c:40959 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T06:16:50,324 INFO [M:0;6336e8474e1c:40959 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40959 2024-12-06T06:16:50,325 INFO [M:0;6336e8474e1c:40959 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T06:16:50,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T06:16:50,427 INFO [M:0;6336e8474e1c:40959 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T06:16:50,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40959-0x1018668262d0000, quorum=127.0.0.1:55017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T06:16:50,432 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T06:16:50,434 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T06:16:50,434 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T06:16:50,434 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T06:16:50,435 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/hadoop.log.dir/,STOPPED} 2024-12-06T06:16:50,438 WARN [BP-840892825-172.17.0.2-1733465616971 heartbeating to localhost/127.0.0.1:37641 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T06:16:50,438 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T06:16:50,438 WARN [BP-840892825-172.17.0.2-1733465616971 heartbeating to localhost/127.0.0.1:37641 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-840892825-172.17.0.2-1733465616971 (Datanode Uuid b200ec0b-a77d-4fc5-b349-d290ec2f1a55) service to localhost/127.0.0.1:37641 2024-12-06T06:16:50,438 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T06:16:50,440 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/cluster_f2665540-2ae4-21bf-204f-a0e90cc9380d/data/data1/current/BP-840892825-172.17.0.2-1733465616971 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T06:16:50,440 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/cluster_f2665540-2ae4-21bf-204f-a0e90cc9380d/data/data2/current/BP-840892825-172.17.0.2-1733465616971 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T06:16:50,441 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T06:16:50,450 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T06:16:50,451 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T06:16:50,451 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T06:16:50,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T06:16:50,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f5866168-05f6-2313-518b-f4d6cb702c2c/hadoop.log.dir/,STOPPED} 2024-12-06T06:16:50,468 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T06:16:50,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-06T06:16:50,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T06:16:50,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T06:16:50,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-06T06:16:50,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees